Imports

In [108]:
#!/usr/bin/env python

from __future__ import print_function

import platform
import sys
import os

import warnings
warnings.simplefilter(action='ignore')

import logging
#print all logging.INFO details
#logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
#Supress default INFO logging
logger = logging.getLogger()
logger.setLevel(logging.CRITICAL)

from time import time
from itertools import groupby
from  more_itertools import unique_everseen
from operator import itemgetter

import bs4
from bs4 import BeautifulSoup
import urllib
from urllib.request import urlopen

import re
import regex
import string
import textwrap
from textwrap import fill

#import pycontractions
#from pycontractions import Contractions
import autocorrect
from autocorrect import spell

import numpy as np
import pandas as pd

import textacy
from textacy.preprocess import remove_punct

import spacy
from spacy.tokenizer import Tokenizer

import gensim
from gensim.matutils import hellinger
from gensim import corpora, models, similarities
from gensim.corpora import Dictionary
from gensim.models import Doc2Vec, CoherenceModel, LdaModel, HdpModel, LsiModel
from gensim.models.wrappers import LdaMallet #,LdaVowpalWabbit

import lda

import pyLDAvis.gensim
pyLDAvis.enable_notebook()

import IPython
from IPython.display import display

import sklearn
from sklearn import metrics
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, TfidfTransformer
from sklearn.preprocessing import normalize
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA, NMF, TruncatedSVD #,LatentDirichletAllocation
from sklearn.manifold import TSNE

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec
%matplotlib inline

import plotly
from plotly import tools
import plotly.plotly as py
import plotly.graph_objs as go
import plotly.figure_factory as ff
from plotly.offline import download_plotlyjs,init_notebook_mode,plot,iplot
#connects JS to notebook so plots work inline
init_notebook_mode(connected=True)

import cufflinks as cf
#allow offline use of cufflinks
cf.go_offline()

import bokeh
from bokeh.io import push_notebook, show, output_notebook
import bokeh.plotting as bp
from bokeh.plotting import figure, save
from bokeh.models import ColumnDataSource, LabelSet, HoverTool
output_notebook()
Loading BokehJS ...

Environment and Library Information

In [3]:
#environment and package versions
print('\n')
print("_"*70)
print('The environment and package versions used in this script are:')
print('\n')

print(platform.platform())
print('Python', sys.version)
print('OS', os.name)
print('Beautiful Soup', bs4.__version__)
print('Urllib', urllib.request.__version__) 
print('Regex', re.__version__)
print('Numpy', np.__version__)
print('Pandas', pd.__version__)
print('Textacy', textacy.__version__)
print('SpaCy', spacy.__version__)
print('Gensim', gensim.__version__)
print('LDA', lda.__version__)
print('PyLDAvis', pyLDAvis.__version__)
print('IPython', IPython.__version__)
print('Sklearn', sklearn.__version__)
print('Matplotlib', matplotlib.__version__)
print('Plotly', plotly.__version__)
print('Cufflinks', cf.__version__)
print('Bokeh', bokeh.__version__)

print('\n')
print("~"*70)
print('\n')

______________________________________________________________________
The environment and package versions used in this script are:


Windows-10-10.0.17134-SP0
Python 3.6.6 |Anaconda custom (64-bit)| (default, Jun 28 2018, 11:27:44) [MSC v.1900 64 bit (AMD64)]
OS nt
Beautiful Soup 4.6.0
Urllib 3.6
Regex 2.2.1
Numpy 1.15.4
Pandas 0.22.0
Textacy 0.6.2
SpaCy 2.0.12
Gensim 3.6.0
LDA 1.1.0
PyLDAvis 2.1.2
IPython 5.1.0
Sklearn 0.19.1
Matplotlib 2.2.2
Plotly 2.2.2
Cufflinks 0.12.1
Bokeh 0.12.4


~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~


Helper Functions

In [4]:
def get_movie_reviews(soup_broth, n_reviews_per_movie=1):
    
    print('Retrieving all baseline URLs... \n')

    base_urls = [ ("https://www.imdb.com" + tag.attrs['href'], tag.text.replace('\n',' ') ) 
                            for tag in soup_broth.findAll('a', attrs={'href': re.compile("^/title/.*_tt")}) ]
    
    print('Retrieved all second-level URLs... \n')

    level_2_urls = []
    for url, title in base_urls:
        soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
        update_url = [("https://www.imdb.com" + tag.attrs['href']) 
                            for tag in soup.findAll('a', attrs={'href': re.compile("^/title/.*tt_urv")})]
        level_2_urls.append((update_url[0], title))
        
    print('Retrieved all third-level URLs... \n')

    level_3_urls = []
    for url, title in level_2_urls:
        soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
        update_url = [("https://www.imdb.com" + tag.attrs['href'])
                          for tag in soup.findAll('a', attrs={'href': re.compile("^/review/.*tt_urv")})[:(n_reviews_per_movie*2)]]
        update_url = list(unique_everseen(update_url))
        for i in update_url:
            level_3_urls.append((i, title))
        
    print('Retrieved all fourth-level URLs... \n')

    level_4_urls = []
    for url, title in level_3_urls:
        soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
        update_url = [("https://www.imdb.com" + soup.find('a', href=re.compile("^/review/.*rw_urv"))['href'])]
        level_4_urls.append((update_url[0], title))
        
    print('Retrieved all fifth-level URLs... \n')

    level_5_text = []
    for url, title in level_4_urls:
        soup = BeautifulSoup(urlopen(url).read(), 'html.parser')
        review_text = [(soup.find('div', {'class' : re.compile("^text")}).text)]
        level_5_text.append((review_text[0], title))
        
    print('All reviews retrieved! \n')

    return level_5_text
In [5]:
def clean_component(review, stop_words, tokenizer, puncts):
    """Text Cleaner: Tokenize, Remove Stopwords, Punctuation, Lemmatize, Spell Correct, Lowercase"""
    
    #rev_contract_exp = list(contract_model.expand_texts([review], precise=True))
    
    doc_tok = tokenizer(review)

    doc_lems = [tok.lemma_ for tok in doc_tok 
                    if (tok.text not in stop_words
                        and tok.text not in puncts
                        and tok.pos_ != "PUNCT" and tok.pos_ != "SYM")]
    
    lem_list = [re.search(r'\(?([0-9A-Za-z-]+)\)?', tok).group(1) 
                    if '-' in tok 
                    else spell(remove_punct(tok)) 
                        for tok in doc_lems]

    doc2vec_input = [t.lower() for tok in lem_list 
                         for t in tok.split() 
                             if t.lower() not in stop_words]
    
    return doc2vec_input
In [6]:
def get_tagged_documents(input_review_texts, stop_words, tokenizer, puncts, sentence_labeler):
    print('Creating Tagged Documents... \n')
    
    all_content = []
    j=0
    for rev, ttl in input_review_texts:
        print('Cleaning review #{}'.format(j+1))
        clean_rev = clean_component(rev, stop_words, tokenizer, puncts)
        all_content.append(sentence_labeler(clean_rev, [ttl]))
        j += 1

    print('Total Number of Movie Review Document Vectors: ', j)
    
    return all_content
In [39]:
def compute_coherence_values(model_type, dictionary, texts, max_topics, 
                                 corpus=False, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                 min_topics=2, stride=3, n_top_words=False, measure='u_mass',
                                 lsi_flag=False, nmf_flag=False, mallet_flag=False, mallet_path=False):

    coherence_values = []
    model_list = []
    if nmf_flag:
        feat_names = transformed_vectorizer.get_feature_names()
    
    for num_topics in range(min_topics, max_topics, stride):
        if lsi_flag:
            model=model_type(corpus=corpus, id2word=dictionary, num_topics=num_topics)
        elif nmf_flag:
            model=model_type(n_components=num_topics, init='nndsvd', random_state=111)
        elif mallet_flag:
            model=model_type(mallet_path=mallet_path, corpus=corpus, id2word=dictionary, num_topics=num_topics) 
        else:
            model=model_type(corpus=corpus, id2word=dictionary, num_topics=num_topics, random_state=121)   
            
        model_list.append(model)
        
        if nmf_flag:
            words_list = []
            
            for i in range(num_topics):
                model.fit(tfidf_norm)
                
                #for each topic, obtain the largest values, and add the words they map to into the dictionary
                words_ids = model.components_[i].argsort()[:-n_top_words - 1:-1]
                words = [feat_names[key] for key in words_ids]
                words_list.append(words)
                
            coherencemodel = CoherenceModel(topics=words_list, texts=texts, dictionary=dictionary, coherence=measure)
            
        else:
            coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence=measure)
        
        coherence_values.append(coherencemodel.get_coherence())
        print('Coherence of model with {} topics has been computed'.format(num_topics))
        
    print('All coherence values have been computed for the {} using the {} measure'.format(model_type, measure.upper()))
    print('\n')

    return model_list, coherence_values
In [8]:
def show_best_num_topics(model_type, umass_coherence_vals, cv_coherence_vals, max_topics, min_topics=2, stride=3):
    
    min_topics=min_topics
    max_topics=max_topics
    stride=stride
    
    x = range(min_topics, max_topics, stride)
    
    max_y1 = max(umass_coherence_vals)
    max_x1 = x[umass_coherence_vals.index(max_y1)]
    
    max_y2 = max(cv_coherence_vals)
    max_x2 = x[cv_coherence_vals.index(max_y2)] 
    
    fig = plt.figure(constrained_layout=True, figsize=(14,5))

    gs = GridSpec(1, 2, figure=fig)
    ax1 = fig.add_subplot(gs[0, 0])
    ax2 = fig.add_subplot(gs[0, -1])

    ax1.plot(x, umass_coherence_vals, label='Coherence Values')
    ax1.set_xlabel('Num Topics')
    ax1.set_ylabel('Coherence score (U_MASS)')
    ax1.legend(loc='best')
    ax1.text(max_x1, max_y1, str((max_x1, max_y1)))

    ax2.plot(x, cv_coherence_vals, label='Coherence Values')
    ax2.set_xlabel('Num Topics')
    ax2.set_ylabel('Coherence score (C_V)')
    ax2.legend(loc='best')
    ax2.text(max_x2, max_y2, str((max_x2, max_y2)))

    fig.suptitle('Coherence Scores ({})'.format(model_type))

    plt.show()

    print('The most coherent number of {} topics using the U_MASS measure is: {}'.format(model_type, max_x1))
    print('The most coherent number of {} topics using the C_V measure is: {}'.format(model_type, max_x2))
    print('\n')
    
    tup_list = [(max_x1, max_y1), (max_x2, max_y2)]
    best_num_topics = max(tup_list, key=itemgetter(1))[0]
    
    return best_num_topics
In [9]:
def get_topics(model, num_topics):
    
    word_dict = {}
    for i in range(num_topics):
        words = model.show_topic(i, topn = 20)
        word_dict['Topic # ' + '{:02d}'.format(i+1)] = [i[0] for i in words]
        
    return pd.DataFrame(word_dict)
In [10]:
def get_nmf_topics(model, num_topics, n_top_words):
    
    #the word ids obtained need to be reverse-mapped to the words so we can print the topic names.
    feat_names = vectorizer.get_feature_names()
    
    word_dict = {}
    for i in range(num_topics):
        
        #for each topic, obtain the largest values, and add the words they map to into the dictionary.
        words_ids = model.components_[i].argsort()[:-n_top_words - 1:-1]
        words = [feat_names[key] for key in words_ids]
        word_dict['Topic # ' + '{:02d}'.format(i+1)] = words
    
    return pd.DataFrame(word_dict)
In [11]:
def topics_2_bow(topic, model, lsi_flag=False, mallet_flag=False):

    topic = topic.split('+')
    topic_bow = []
    
    lsi_mallet_array = np.array([])
    lsi_mallet_dict = {}
    lsi_mallet_dict_scaled = {}
    
    for word in topic:
        #split probability and word
        try:
            prob, word = word.split('*')
        except:
            continue

        #replace unwanted characters
        rep = {' ': '', '"': ''}
        replace = dict((re.escape(k), v) for k, v in rep.items())
        pattern = re.compile("|".join(replace.keys()))
        word = pattern.sub(lambda m: replace[re.escape(m.group(0))], word)

        #convert to word_type
        try:
            word = model.id2word.doc2bow([word])[0][0]
        except:
            continue
            
        if lsi_flag or mallet_flag:
            lsi_mallet_array = np.append(lsi_mallet_array, float(prob))
            lsi_mallet_dict.update({word:prob})
              
        else:
            topic_bow.append((word, float(prob)))
                        
    if lsi_flag or mallet_flag:
        scaler = MinMaxScaler(feature_range=(0,1), copy=True)
        lsi_mallet_scaled = scaler.fit_transform(lsi_mallet_array.reshape(-1,1))
        
        lsi_mallet_scaled = (lsi_mallet_scaled - lsi_mallet_scaled.min()) / (lsi_mallet_scaled - lsi_mallet_scaled.min()).sum()
        
        for k,v in zip(lsi_mallet_dict.keys(), lsi_mallet_scaled):
            lsi_mallet_dict_scaled.update({k:v[0]})        

        for k,v in lsi_mallet_dict_scaled.items():
            topic_bow.append((k,v))
        
    return topic_bow
In [12]:
def get_most_similar_topics(model, topics_df=False, num_topics=False, mallet_flag=False, hdp_flag=False, lsi_flag=False, nmf_flag=False, columns=False):
    
    if not nmf_flag:
    
        mod_topics = tuple(topics_df.columns)
        mod_top_dict = {}

        if hdp_flag:
            for k,v in zip(mod_topics, model.show_topics(num_words=len(model.id2word))):
                mod_top_dict.update({k:v})
        else:    
            for k,v in zip(mod_topics, model.show_topics(num_words=len(model.id2word), num_topics=num_topics)):
                mod_top_dict.update({k:v})

        for k,v in mod_top_dict.items():
            mod_top_dict[k] = topics_2_bow(v[1], model, lsi_flag, mallet_flag)
    
    else:
        mod_topics = tuple(columns)
        mod_top_dict = {}
        
        nmf_top_list = []
        for k,v in zip(mod_topics, model.components_):
            nmf_top_list.append(tuple((k, v)))
        
        scaler = MinMaxScaler(feature_range=(0,1), copy=True)
        
        for k,v in nmf_top_list:
            v_scaled = scaler.fit_transform(v.reshape(-1,1))
            v = (v_scaled - v_scaled.min()) / (v_scaled - v_scaled.min()).sum()
            mod_top_dict.update({k:v})

    hellinger_dists = [(hellinger(mod_top_dict[x], mod_top_dict[y]), x, y)
                          for i,x in enumerate(mod_top_dict.keys())
                          for j,y in enumerate(mod_top_dict.keys())
                          if i != j]       

    unique_hellinger = [tuple(x) for x in set(map(frozenset, hellinger_dists)) if len(tuple(x)) == 3]
    
    resorted_hellinger = []
    for i in range(len(unique_hellinger)):
        resorted_hellinger.append(sorted(tuple(str(e) for e in unique_hellinger[i])))
        resorted_hellinger[i][0] = float(resorted_hellinger[i][0])
        resorted_hellinger[i] = tuple(resorted_hellinger[i])

    resorted_hellinger = sorted(resorted_hellinger, reverse=True)

    return resorted_hellinger
In [13]:
def get_dominant_topics(model, corpus, texts, lsi_flag=False):

    dom_topics_df = pd.DataFrame()

    #for all topic/topic-probability pairings
    for i, row in enumerate(model[corpus]):
        #return the pairings, sorting first the one with the highest topic-probability in the mixture
        if lsi_flag:
            row = sorted(row, key=lambda x: abs(x[1]), reverse=True)
        else:
            row = sorted(row, key=lambda x: (x[1]), reverse=True)     
        #for every topic/topic-probability pairing in the sorted tuple list
        for j, (topic_num, topic_prob) in enumerate(row):
            #take the pairing with the highest topic-probability
            if j == 0:
                #return the tuple list of top 'n' word-probabilities associated with that topic
                wp = model.show_topic(topic_num, topn=20)
                #create a list of those top 'n' words
                topic_keywords = ", ".join([word for word, prob in wp])
                #append to the empty dataframe a series containing:
                #    the dominant topic allocation for that document,
                #    the topic-probability it contributes to that document,
                #    and the top 'n' words associated with that topic
                dom_topics_df = dom_topics_df.append(pd.Series([int(topic_num), round(topic_prob,4), topic_keywords]), ignore_index=True)
            else:
                #ignore other topics in the mixture
                break
                
    #name the columns of the constructed dataframe
    dom_topics_df.columns = ['Dominant_Topic', 'Probability_Contribution', 'Topic_Keywords']

    #append to the original text as another column in the dataframe
    contents = pd.Series(texts)
    dom_topics_df = pd.concat([dom_topics_df, contents], axis=1)
    
    dom_topics_final = dom_topics_df.reset_index()
    dom_topics_final.columns = ['Document_Number', 'Dominant_Topic', 'Probability_Contribution', 'Topic_Keywords', 'Original_Text']
    dom_topics_final.set_index('Document_Number', inplace=True)
    
    return dom_topics_df, dom_topics_final
In [14]:
def get_most_representative_docs(dom_topics_final_df, n_topics=20, lsi_flag=False):
    
    representative_docs = pd.DataFrame()
    
    dom_topics_grpd = dom_topics_final_df.groupby('Dominant_Topic')

    #for every dominant topic, find the document to which it contributed the greatest (or largest magnitude [LSI]) probability contribution
    if not lsi_flag:    
        for i, grp in dom_topics_grpd:
            representative_docs = pd.concat([representative_docs, 
                                                grp.sort_values(['Probability_Contribution'], ascending=True).head(1)], 
                                                axis=0)
    else:
        for i, grp in dom_topics_grpd:
            representative_docs = pd.concat([representative_docs,
                                                #sort by descending absolute value
                                                grp.reindex(grp.Probability_Contribution.abs().sort_values(inplace=False, ascending=False).index).head(1)],
                                                axis=0)

    representative_docs.reset_index(drop=True, inplace=True)

    representative_docs.columns = ['Topic_Number', 'Probability_Contribution', 'Topic_Keywords', 'Most_Representative_Document']
    representative_docs['Topic_Number'] = representative_docs['Topic_Number'].astype(int)

    representative_docs.set_index('Topic_Number', inplace=True)

    rep_docs_first_n = representative_docs.head(n_topics)
    
    return rep_docs_first_n
In [15]:
def get_topic_distribution(dom_topics_final_df, rep_doc_df=False, n_topics=20, hdp_flag=False):
    
    if hdp_flag:
        
        rep_doc_df = get_most_representative_docs(dom_topics_final_df, n_topics=dom_topics_final_df['Dominant_Topic'].nunique())
    
    rep_doc_df.reset_index(inplace=True)
    
    #topic number and keywords
    topic_num_keywords = rep_doc_df[['Topic_Number', 'Topic_Keywords']]
    
    #number of documents allocated to each topic
    topic_counts = dom_topics_final_df['Dominant_Topic'].value_counts()
    
    #topic allocation percentage of total corpus
    topic_percent = round(topic_counts/topic_counts.sum(), 4)
    
    #concat number of allocated docs and percentage
    topic_dist = pd.concat([topic_num_keywords, topic_counts, topic_percent], axis=1)
    
    topic_dist.columns = ['Topic_Number', 'Topic_Keywords', 'Documents_per_Topic', 'Percent_of_Total_Corpus']
    topic_dist.dropna(axis=0, how='any', inplace=True)
    
    topic_dist['Topic_Number'] = topic_dist['Topic_Number'].astype(int)
    topic_dist.set_index('Topic_Number', inplace=True)
    
    topic_dist = topic_dist[['Documents_per_Topic', 'Percent_of_Total_Corpus', 'Topic_Keywords']]

    topic_dist_first_n = topic_dist.head(n_topics)
    
    return topic_dist_first_n
In [16]:
def svd_2D_scatter(svd_2D_transformed, color_scale_dimension=0):
    
    trace = go.Scattergl(
    x = svd_2D_transformed[:,0],
    y = svd_2D_transformed[:,1],
    mode = 'markers',
    marker = dict(
        color = svd_2D_transformed[:,color_scale_dimension],
        colorscale='Viridis',
        colorbar=dict(title='TF-IDF Variation: SVD Component #{}'.format(color_scale_dimension+1)),
        line = dict(width = 1)
    ),
    text = vectorizer.get_feature_names(),
    )
    
    data = [trace]
    iplot(data, filename='scatter-mode')
    
    return None
In [17]:
def svd_3D_scatter(svd_3D_transformed, color_scale_dimension=0):
    
    svd_3D_dim_range = np.stack((np.amax(svd_3D_transformed, axis=0), np.amin(svd_3D_transformed, axis=0)))
    svd_3D_max_dim = max(np.max(svd_3D_dim_range, axis=0))
    svd_3D_max_dim += svd_3D_max_dim*(0.1)
    
    fig = tools.make_subplots(rows=1, cols=1,
                          print_grid=False,
                          specs=[[{'is_3d': True}]])
    scene = dict(
        camera = dict(
        up=dict(x=0, y=0, z=1),
        center=dict(x=0, y=0, z=0),
        eye=dict(x=2.5, y=0.1, z=0.1)
        ),

        xaxis=dict(
            range=[-svd_3D_max_dim, svd_3D_max_dim],
            title='SVD_C1',
            gridcolor='rgb(255, 255, 255)',
            zerolinecolor='rgb(255, 255, 255)',
            showbackground=True,
            backgroundcolor='rgb(230, 230,230)',
            showticklabels=False, ticks=''
        ),
        yaxis=dict(
            range=[-svd_3D_max_dim, svd_3D_max_dim],
            title='SVD_C2',
            gridcolor='rgb(255, 255, 255)',
            zerolinecolor='rgb(255, 255, 255)',
            showbackground=True,
            backgroundcolor='rgb(230, 230,230)',
            showticklabels=False, ticks=''
        ),
        zaxis=dict(
            range=[-svd_3D_max_dim, svd_3D_max_dim],
            title='SVD_C3',
            gridcolor='rgb(255, 255, 255)',
            zerolinecolor='rgb(255, 255, 255)',
            showbackground=True,
            backgroundcolor='rgb(230, 230,230)',
            showticklabels=False, ticks=''
        )
    )

    centers = [[1, 1], [-1, -1], [1, -1]]
    X = svd_3D_transformed

    trace = go.Scatter3d(x=X[:, 0], y=X[:, 1], z=X[:, 2],
                             showlegend=False,
                             mode='markers',

                             marker=dict(
                                color=X[:,color_scale_dimension],
                                colorscale='Viridis',
                                colorbar=dict(title='TF-IDF Variation: SVD Component #{}'.format(color_scale_dimension+1)),
                                line=dict(color='black', width=1)),

                            text = vectorizer.get_feature_names())

    fig.append_trace(trace, 1, 1)

    fig['layout'].update(height=1000, width=1200,
                         margin=dict(l=10,r=10))

    fig['layout']['scene'].update(scene)

    iplot(fig)
    
    return None
In [18]:
def plot_tsne(doc_list, fitted_lda, fitted_count_vectorizer, transformed_lda, transformed_tsne, color_map, n_top_words=10):
    
    n_top_words = n_top_words
    color_map = color_map
    
    #retrieve component key words
    _lda_keys = []
    for i in range(transformed_lda.shape[0]):
        _lda_keys +=  transformed_lda[i].argmax(),
        
    topic_summaries = []
    #matrix of shape n_topics x len(vocabulary)
    topic_words = fitted_lda.topic_word_
    #all vocab words (strings)
    vocab = fitted_count_vectorizer.get_feature_names()
    for i, topic_dist in enumerate(topic_words):
        #np.argsort returns indices that would sort an array
        #iterates over topic component vectors, sorts array in asc order, appends key words from end of array
        topic_word = np.array(vocab)[np.argsort(topic_dist)][:-(n_top_words + 1):-1]
        topic_summaries.append(' '.join(topic_word))
    
    num_example = len(transformed_lda)

    plot_dict = {
            'x': transformed_tsne[:, 0],
            'y': transformed_tsne[:, 1],
            'colors': color_map[_lda_keys][:num_example],
            'content': doc_list[:num_example],
            'topic_key': _lda_keys[:num_example]
            }

    #create dataframe from dictionary
    plot_df = pd.DataFrame.from_dict(plot_dict)

    source = bp.ColumnDataSource(data=plot_df)
    title = 'LDA T-SNE Visualization'

    plot_lda = bp.figure(plot_width=1400, plot_height=1100,
                         title=title,
                         tools="pan,wheel_zoom,box_zoom,reset,hover,previewsave",
                         x_axis_type=None, y_axis_type=None, min_border=1)

    plot_lda.scatter('x','y', color='colors', source=source)

    topic_coord = np.empty((transformed_lda.shape[1], 2)) * np.nan
    for topic_num in _lda_keys:
        if not np.isnan(topic_coord).any():
            break
        topic_coord[topic_num] = transformed_tsne[_lda_keys.index(topic_num)]

    #plot key words
    for i in range(transformed_lda.shape[1]):
        plot_lda.text(topic_coord[i, 0], topic_coord[i, 1], [topic_summaries[i]])

    #hover tools
    hover = plot_lda.select(dict(type=HoverTool))
    hover.tooltips = {"content": "@content - topic: @topic_key"}

    # save the plot
    #save(plot_lda, '{}.html'.format(title))

    #Cf. JSON Serialization issue: 
    #    https://github.com/bokeh/bokeh/issues/5439
    #    https://github.com/bokeh/bokeh/issues/6222
    #   https://github.com/bokeh/bokeh/issues/7523
    try:
        show(plot_lda, notebook_handle=True)
    except Exception as e:
        print('Note!: {}'.format(e.__doc__))
        print(e)

    return None
In [19]:
#dataframe styling
def color_green(val):
    color = 'green' if isinstance(val, float) else 'black'
    return 'color: {col}'.format(col=color)

def make_bold(val):
    weight = 700 if isinstance(val, float) else 400
    return 'font-weight: {wt}'.format(wt=weight)
In [20]:
def pretty_print(input_text):
    
    pieces = [str(word) for word in input_text]
    output = ' '.join(pieces)
    #write_up = fill(output)
    print('Hellinger Distances between topics are: \n')
    print('_'*70)
    print('\n')
    print(output.translate({ord(')'):')\n'}))
    print('_'*70)
    #print(write_up)
    
    return None

Loading Language, Punctuation, and Tagged Document Models

In [21]:
nlp = spacy.load('en')

#cont = Contractions('../GoogleNews-vectors-negative300.bin.gz')
#cont.load_models()

punctuations = string.punctuation

LabeledSentence1 = gensim.models.doc2vec.TaggedDocument

Loading Tokenizer and Updating Stop Words

In [22]:
tokenizer = Tokenizer(nlp.vocab)

#stopwords = spacy.lang.en.STOP_WORDS
#spacy.lang.en.STOP_WORDS.add("e.g.")
#nlp.vocab['the'].is_stop
nlp.Defaults.stop_words |= {"(a)", "(b)", "(c)", "etc", "etc.", "etc.)", "w/e", "(e.g.", "no?", "s", 
                           "film", "movie","0","1","2","3","4","5","6","7","8","9","10","e","f","k","n","q",
                            "de","oh","ones","miike","http","imdb", "horror", "like", "good", "great", "little", 
                            "come", "way", "know", "michael", "lot", "thing", "films", "later", "actually", "find", 
                            "big", "long", "away", "filmthe", "www", "com", "x", "aja", "agritos", "lon", "therebravo", 
                            "gou", "b", "particularly", "probably", "sure", "greenskeeper", "try", 
                            "half", "intothe", "especially", "exactly", "20", "ukr", "thatll", "darn", "certainly", "simply", }

stopwords = list(nlp.Defaults.stop_words)

Retrieving and Cleaning Reviews of the Best Horror Movies

In [24]:
#open the base URL webpage
html_page = urlopen("https://www.imdb.com/list/ls059633855/")

#instantiate beautiful soup object of the html page
soup = BeautifulSoup(html_page, 'lxml')

review_text_first_5 = get_movie_reviews(soup, n_reviews_per_movie=5)
Retrieving all baseline URLs... 

Retrieved all second-level URLs... 

Retrieved all third-level URLs... 

Retrieved all fourth-level URLs... 

Retrieved all fifth-level URLs... 

All reviews retrieved! 

In [27]:
all_good_movie_reviews_500 = get_tagged_documents(review_text_first_5, stopwords, tokenizer, punctuations, LabeledSentence1)
Creating Tagged Documents... 

Cleaning review #1
Cleaning review #2
Cleaning review #3
Cleaning review #4
Cleaning review #5
Cleaning review #6
Cleaning review #7
Cleaning review #8
Cleaning review #9
Cleaning review #10
Cleaning review #11
Cleaning review #12
Cleaning review #13
Cleaning review #14
Cleaning review #15
Cleaning review #16
Cleaning review #17
Cleaning review #18
Cleaning review #19
Cleaning review #20
Cleaning review #21
Cleaning review #22
Cleaning review #23
Cleaning review #24
Cleaning review #25
Cleaning review #26
Cleaning review #27
Cleaning review #28
Cleaning review #29
Cleaning review #30
Cleaning review #31
Cleaning review #32
Cleaning review #33
Cleaning review #34
Cleaning review #35
Cleaning review #36
Cleaning review #37
Cleaning review #38
Cleaning review #39
Cleaning review #40
Cleaning review #41
Cleaning review #42
Cleaning review #43
Cleaning review #44
Cleaning review #45
Cleaning review #46
Cleaning review #47
Cleaning review #48
Cleaning review #49
Cleaning review #50
Cleaning review #51
Cleaning review #52
Cleaning review #53
Cleaning review #54
Cleaning review #55
Cleaning review #56
Cleaning review #57
Cleaning review #58
Cleaning review #59
Cleaning review #60
Cleaning review #61
Cleaning review #62
Cleaning review #63
Cleaning review #64
Cleaning review #65
Cleaning review #66
Cleaning review #67
Cleaning review #68
Cleaning review #69
Cleaning review #70
Cleaning review #71
Cleaning review #72
Cleaning review #73
Cleaning review #74
Cleaning review #75
Cleaning review #76
Cleaning review #77
Cleaning review #78
Cleaning review #79
Cleaning review #80
Cleaning review #81
Cleaning review #82
Cleaning review #83
Cleaning review #84
Cleaning review #85
Cleaning review #86
Cleaning review #87
Cleaning review #88
Cleaning review #89
Cleaning review #90
Cleaning review #91
Cleaning review #92
Cleaning review #93
Cleaning review #94
Cleaning review #95
Cleaning review #96
Cleaning review #97
Cleaning review #98
Cleaning review #99
Cleaning review #100
Cleaning review #101
Cleaning review #102
Cleaning review #103
Cleaning review #104
Cleaning review #105
Cleaning review #106
Cleaning review #107
Cleaning review #108
Cleaning review #109
Cleaning review #110
Cleaning review #111
Cleaning review #112
Cleaning review #113
Cleaning review #114
Cleaning review #115
Cleaning review #116
Cleaning review #117
Cleaning review #118
Cleaning review #119
Cleaning review #120
Cleaning review #121
Cleaning review #122
Cleaning review #123
Cleaning review #124
Cleaning review #125
Cleaning review #126
Cleaning review #127
Cleaning review #128
Cleaning review #129
Cleaning review #130
Cleaning review #131
Cleaning review #132
Cleaning review #133
Cleaning review #134
Cleaning review #135
Cleaning review #136
Cleaning review #137
Cleaning review #138
Cleaning review #139
Cleaning review #140
Cleaning review #141
Cleaning review #142
Cleaning review #143
Cleaning review #144
Cleaning review #145
Cleaning review #146
Cleaning review #147
Cleaning review #148
Cleaning review #149
Cleaning review #150
Cleaning review #151
Cleaning review #152
Cleaning review #153
Cleaning review #154
Cleaning review #155
Cleaning review #156
Cleaning review #157
Cleaning review #158
Cleaning review #159
Cleaning review #160
Cleaning review #161
Cleaning review #162
Cleaning review #163
Cleaning review #164
Cleaning review #165
Cleaning review #166
Cleaning review #167
Cleaning review #168
Cleaning review #169
Cleaning review #170
Cleaning review #171
Cleaning review #172
Cleaning review #173
Cleaning review #174
Cleaning review #175
Cleaning review #176
Cleaning review #177
Cleaning review #178
Cleaning review #179
Cleaning review #180
Cleaning review #181
Cleaning review #182
Cleaning review #183
Cleaning review #184
Cleaning review #185
Cleaning review #186
Cleaning review #187
Cleaning review #188
Cleaning review #189
Cleaning review #190
Cleaning review #191
Cleaning review #192
Cleaning review #193
Cleaning review #194
Cleaning review #195
Cleaning review #196
Cleaning review #197
Cleaning review #198
Cleaning review #199
Cleaning review #200
Cleaning review #201
Cleaning review #202
Cleaning review #203
Cleaning review #204
Cleaning review #205
Cleaning review #206
Cleaning review #207
Cleaning review #208
Cleaning review #209
Cleaning review #210
Cleaning review #211
Cleaning review #212
Cleaning review #213
Cleaning review #214
Cleaning review #215
Cleaning review #216
Cleaning review #217
Cleaning review #218
Cleaning review #219
Cleaning review #220
Cleaning review #221
Cleaning review #222
Cleaning review #223
Cleaning review #224
Cleaning review #225
Cleaning review #226
Cleaning review #227
Cleaning review #228
Cleaning review #229
Cleaning review #230
Cleaning review #231
Cleaning review #232
Cleaning review #233
Cleaning review #234
Cleaning review #235
Cleaning review #236
Cleaning review #237
Cleaning review #238
Cleaning review #239
Cleaning review #240
Cleaning review #241
Cleaning review #242
Cleaning review #243
Cleaning review #244
Cleaning review #245
Cleaning review #246
Cleaning review #247
Cleaning review #248
Cleaning review #249
Cleaning review #250
Cleaning review #251
Cleaning review #252
Cleaning review #253
Cleaning review #254
Cleaning review #255
Cleaning review #256
Cleaning review #257
Cleaning review #258
Cleaning review #259
Cleaning review #260
Cleaning review #261
Cleaning review #262
Cleaning review #263
Cleaning review #264
Cleaning review #265
Cleaning review #266
Cleaning review #267
Cleaning review #268
Cleaning review #269
Cleaning review #270
Cleaning review #271
Cleaning review #272
Cleaning review #273
Cleaning review #274
Cleaning review #275
Cleaning review #276
Cleaning review #277
Cleaning review #278
Cleaning review #279
Cleaning review #280
Cleaning review #281
Cleaning review #282
Cleaning review #283
Cleaning review #284
Cleaning review #285
Cleaning review #286
Cleaning review #287
Cleaning review #288
Cleaning review #289
Cleaning review #290
Cleaning review #291
Cleaning review #292
Cleaning review #293
Cleaning review #294
Cleaning review #295
Cleaning review #296
Cleaning review #297
Cleaning review #298
Cleaning review #299
Cleaning review #300
Cleaning review #301
Cleaning review #302
Cleaning review #303
Cleaning review #304
Cleaning review #305
Cleaning review #306
Cleaning review #307
Cleaning review #308
Cleaning review #309
Cleaning review #310
Cleaning review #311
Cleaning review #312
Cleaning review #313
Cleaning review #314
Cleaning review #315
Cleaning review #316
Cleaning review #317
Cleaning review #318
Cleaning review #319
Cleaning review #320
Cleaning review #321
Cleaning review #322
Cleaning review #323
Cleaning review #324
Cleaning review #325
Cleaning review #326
Cleaning review #327
Cleaning review #328
Cleaning review #329
Cleaning review #330
Cleaning review #331
Cleaning review #332
Cleaning review #333
Cleaning review #334
Cleaning review #335
Cleaning review #336
Cleaning review #337
Cleaning review #338
Cleaning review #339
Cleaning review #340
Cleaning review #341
Cleaning review #342
Cleaning review #343
Cleaning review #344
Cleaning review #345
Cleaning review #346
Cleaning review #347
Cleaning review #348
Cleaning review #349
Cleaning review #350
Cleaning review #351
Cleaning review #352
Cleaning review #353
Cleaning review #354
Cleaning review #355
Cleaning review #356
Cleaning review #357
Cleaning review #358
Cleaning review #359
Cleaning review #360
Cleaning review #361
Cleaning review #362
Cleaning review #363
Cleaning review #364
Cleaning review #365
Cleaning review #366
Cleaning review #367
Cleaning review #368
Cleaning review #369
Cleaning review #370
Cleaning review #371
Cleaning review #372
Cleaning review #373
Cleaning review #374
Cleaning review #375
Cleaning review #376
Cleaning review #377
Cleaning review #378
Cleaning review #379
Cleaning review #380
Cleaning review #381
Cleaning review #382
Cleaning review #383
Cleaning review #384
Cleaning review #385
Cleaning review #386
Cleaning review #387
Cleaning review #388
Cleaning review #389
Cleaning review #390
Cleaning review #391
Cleaning review #392
Cleaning review #393
Cleaning review #394
Cleaning review #395
Cleaning review #396
Cleaning review #397
Cleaning review #398
Cleaning review #399
Cleaning review #400
Cleaning review #401
Cleaning review #402
Cleaning review #403
Cleaning review #404
Cleaning review #405
Cleaning review #406
Cleaning review #407
Cleaning review #408
Cleaning review #409
Cleaning review #410
Cleaning review #411
Cleaning review #412
Cleaning review #413
Cleaning review #414
Cleaning review #415
Cleaning review #416
Cleaning review #417
Cleaning review #418
Cleaning review #419
Cleaning review #420
Cleaning review #421
Cleaning review #422
Cleaning review #423
Cleaning review #424
Cleaning review #425
Cleaning review #426
Cleaning review #427
Cleaning review #428
Cleaning review #429
Cleaning review #430
Cleaning review #431
Cleaning review #432
Cleaning review #433
Cleaning review #434
Cleaning review #435
Cleaning review #436
Cleaning review #437
Cleaning review #438
Cleaning review #439
Cleaning review #440
Cleaning review #441
Cleaning review #442
Cleaning review #443
Cleaning review #444
Cleaning review #445
Cleaning review #446
Cleaning review #447
Cleaning review #448
Cleaning review #449
Cleaning review #450
Cleaning review #451
Cleaning review #452
Cleaning review #453
Cleaning review #454
Cleaning review #455
Cleaning review #456
Cleaning review #457
Cleaning review #458
Cleaning review #459
Cleaning review #460
Cleaning review #461
Cleaning review #462
Cleaning review #463
Cleaning review #464
Cleaning review #465
Cleaning review #466
Cleaning review #467
Cleaning review #468
Cleaning review #469
Cleaning review #470
Cleaning review #471
Cleaning review #472
Cleaning review #473
Cleaning review #474
Cleaning review #475
Cleaning review #476
Cleaning review #477
Cleaning review #478
Cleaning review #479
Cleaning review #480
Cleaning review #481
Cleaning review #482
Cleaning review #483
Cleaning review #484
Cleaning review #485
Cleaning review #486
Cleaning review #487
Cleaning review #488
Cleaning review #489
Cleaning review #490
Cleaning review #491
Cleaning review #492
Cleaning review #493
Cleaning review #494
Cleaning review #495
Cleaning review #496
Cleaning review #497
Cleaning review #498
Cleaning review #499
Cleaning review #500
Total Number of Movie Review Document Vectors:  500

Retrieving and Cleaning Reviews of the Worst Horror Movies

In [28]:
#open the base URL webpage
html_page_bad = urlopen("https://www.imdb.com/list/ls059633855/")

#instantiate beautiful soup object of the html page
soup_bad = BeautifulSoup(html_page_bad, 'lxml')

review_text_first_5_bad = get_movie_reviews(soup_bad, n_reviews_per_movie=5)
Retrieving all baseline URLs... 

Retrieved all second-level URLs... 

Retrieved all third-level URLs... 

Retrieved all fourth-level URLs... 

Retrieved all fifth-level URLs... 

All reviews retrieved! 

In [29]:
all_bad_movie_reviews_500 = get_tagged_documents(review_text_first_5_bad, stopwords, tokenizer, punctuations, LabeledSentence1)
Creating Tagged Documents... 

Cleaning review #1
Cleaning review #2
Cleaning review #3
Cleaning review #4
Cleaning review #5
Cleaning review #6
Cleaning review #7
Cleaning review #8
Cleaning review #9
Cleaning review #10
Cleaning review #11
Cleaning review #12
Cleaning review #13
Cleaning review #14
Cleaning review #15
Cleaning review #16
Cleaning review #17
Cleaning review #18
Cleaning review #19
Cleaning review #20
Cleaning review #21
Cleaning review #22
Cleaning review #23
Cleaning review #24
Cleaning review #25
Cleaning review #26
Cleaning review #27
Cleaning review #28
Cleaning review #29
Cleaning review #30
Cleaning review #31
Cleaning review #32
Cleaning review #33
Cleaning review #34
Cleaning review #35
Cleaning review #36
Cleaning review #37
Cleaning review #38
Cleaning review #39
Cleaning review #40
Cleaning review #41
Cleaning review #42
Cleaning review #43
Cleaning review #44
Cleaning review #45
Cleaning review #46
Cleaning review #47
Cleaning review #48
Cleaning review #49
Cleaning review #50
Cleaning review #51
Cleaning review #52
Cleaning review #53
Cleaning review #54
Cleaning review #55
Cleaning review #56
Cleaning review #57
Cleaning review #58
Cleaning review #59
Cleaning review #60
Cleaning review #61
Cleaning review #62
Cleaning review #63
Cleaning review #64
Cleaning review #65
Cleaning review #66
Cleaning review #67
Cleaning review #68
Cleaning review #69
Cleaning review #70
Cleaning review #71
Cleaning review #72
Cleaning review #73
Cleaning review #74
Cleaning review #75
Cleaning review #76
Cleaning review #77
Cleaning review #78
Cleaning review #79
Cleaning review #80
Cleaning review #81
Cleaning review #82
Cleaning review #83
Cleaning review #84
Cleaning review #85
Cleaning review #86
Cleaning review #87
Cleaning review #88
Cleaning review #89
Cleaning review #90
Cleaning review #91
Cleaning review #92
Cleaning review #93
Cleaning review #94
Cleaning review #95
Cleaning review #96
Cleaning review #97
Cleaning review #98
Cleaning review #99
Cleaning review #100
Cleaning review #101
Cleaning review #102
Cleaning review #103
Cleaning review #104
Cleaning review #105
Cleaning review #106
Cleaning review #107
Cleaning review #108
Cleaning review #109
Cleaning review #110
Cleaning review #111
Cleaning review #112
Cleaning review #113
Cleaning review #114
Cleaning review #115
Cleaning review #116
Cleaning review #117
Cleaning review #118
Cleaning review #119
Cleaning review #120
Cleaning review #121
Cleaning review #122
Cleaning review #123
Cleaning review #124
Cleaning review #125
Cleaning review #126
Cleaning review #127
Cleaning review #128
Cleaning review #129
Cleaning review #130
Cleaning review #131
Cleaning review #132
Cleaning review #133
Cleaning review #134
Cleaning review #135
Cleaning review #136
Cleaning review #137
Cleaning review #138
Cleaning review #139
Cleaning review #140
Cleaning review #141
Cleaning review #142
Cleaning review #143
Cleaning review #144
Cleaning review #145
Cleaning review #146
Cleaning review #147
Cleaning review #148
Cleaning review #149
Cleaning review #150
Cleaning review #151
Cleaning review #152
Cleaning review #153
Cleaning review #154
Cleaning review #155
Cleaning review #156
Cleaning review #157
Cleaning review #158
Cleaning review #159
Cleaning review #160
Cleaning review #161
Cleaning review #162
Cleaning review #163
Cleaning review #164
Cleaning review #165
Cleaning review #166
Cleaning review #167
Cleaning review #168
Cleaning review #169
Cleaning review #170
Cleaning review #171
Cleaning review #172
Cleaning review #173
Cleaning review #174
Cleaning review #175
Cleaning review #176
Cleaning review #177
Cleaning review #178
Cleaning review #179
Cleaning review #180
Cleaning review #181
Cleaning review #182
Cleaning review #183
Cleaning review #184
Cleaning review #185
Cleaning review #186
Cleaning review #187
Cleaning review #188
Cleaning review #189
Cleaning review #190
Cleaning review #191
Cleaning review #192
Cleaning review #193
Cleaning review #194
Cleaning review #195
Cleaning review #196
Cleaning review #197
Cleaning review #198
Cleaning review #199
Cleaning review #200
Cleaning review #201
Cleaning review #202
Cleaning review #203
Cleaning review #204
Cleaning review #205
Cleaning review #206
Cleaning review #207
Cleaning review #208
Cleaning review #209
Cleaning review #210
Cleaning review #211
Cleaning review #212
Cleaning review #213
Cleaning review #214
Cleaning review #215
Cleaning review #216
Cleaning review #217
Cleaning review #218
Cleaning review #219
Cleaning review #220
Cleaning review #221
Cleaning review #222
Cleaning review #223
Cleaning review #224
Cleaning review #225
Cleaning review #226
Cleaning review #227
Cleaning review #228
Cleaning review #229
Cleaning review #230
Cleaning review #231
Cleaning review #232
Cleaning review #233
Cleaning review #234
Cleaning review #235
Cleaning review #236
Cleaning review #237
Cleaning review #238
Cleaning review #239
Cleaning review #240
Cleaning review #241
Cleaning review #242
Cleaning review #243
Cleaning review #244
Cleaning review #245
Cleaning review #246
Cleaning review #247
Cleaning review #248
Cleaning review #249
Cleaning review #250
Cleaning review #251
Cleaning review #252
Cleaning review #253
Cleaning review #254
Cleaning review #255
Cleaning review #256
Cleaning review #257
Cleaning review #258
Cleaning review #259
Cleaning review #260
Cleaning review #261
Cleaning review #262
Cleaning review #263
Cleaning review #264
Cleaning review #265
Cleaning review #266
Cleaning review #267
Cleaning review #268
Cleaning review #269
Cleaning review #270
Cleaning review #271
Cleaning review #272
Cleaning review #273
Cleaning review #274
Cleaning review #275
Cleaning review #276
Cleaning review #277
Cleaning review #278
Cleaning review #279
Cleaning review #280
Cleaning review #281
Cleaning review #282
Cleaning review #283
Cleaning review #284
Cleaning review #285
Cleaning review #286
Cleaning review #287
Cleaning review #288
Cleaning review #289
Cleaning review #290
Cleaning review #291
Cleaning review #292
Cleaning review #293
Cleaning review #294
Cleaning review #295
Cleaning review #296
Cleaning review #297
Cleaning review #298
Cleaning review #299
Cleaning review #300
Cleaning review #301
Cleaning review #302
Cleaning review #303
Cleaning review #304
Cleaning review #305
Cleaning review #306
Cleaning review #307
Cleaning review #308
Cleaning review #309
Cleaning review #310
Cleaning review #311
Cleaning review #312
Cleaning review #313
Cleaning review #314
Cleaning review #315
Cleaning review #316
Cleaning review #317
Cleaning review #318
Cleaning review #319
Cleaning review #320
Cleaning review #321
Cleaning review #322
Cleaning review #323
Cleaning review #324
Cleaning review #325
Cleaning review #326
Cleaning review #327
Cleaning review #328
Cleaning review #329
Cleaning review #330
Cleaning review #331
Cleaning review #332
Cleaning review #333
Cleaning review #334
Cleaning review #335
Cleaning review #336
Cleaning review #337
Cleaning review #338
Cleaning review #339
Cleaning review #340
Cleaning review #341
Cleaning review #342
Cleaning review #343
Cleaning review #344
Cleaning review #345
Cleaning review #346
Cleaning review #347
Cleaning review #348
Cleaning review #349
Cleaning review #350
Cleaning review #351
Cleaning review #352
Cleaning review #353
Cleaning review #354
Cleaning review #355
Cleaning review #356
Cleaning review #357
Cleaning review #358
Cleaning review #359
Cleaning review #360
Cleaning review #361
Cleaning review #362
Cleaning review #363
Cleaning review #364
Cleaning review #365
Cleaning review #366
Cleaning review #367
Cleaning review #368
Cleaning review #369
Cleaning review #370
Cleaning review #371
Cleaning review #372
Cleaning review #373
Cleaning review #374
Cleaning review #375
Cleaning review #376
Cleaning review #377
Cleaning review #378
Cleaning review #379
Cleaning review #380
Cleaning review #381
Cleaning review #382
Cleaning review #383
Cleaning review #384
Cleaning review #385
Cleaning review #386
Cleaning review #387
Cleaning review #388
Cleaning review #389
Cleaning review #390
Cleaning review #391
Cleaning review #392
Cleaning review #393
Cleaning review #394
Cleaning review #395
Cleaning review #396
Cleaning review #397
Cleaning review #398
Cleaning review #399
Cleaning review #400
Cleaning review #401
Cleaning review #402
Cleaning review #403
Cleaning review #404
Cleaning review #405
Cleaning review #406
Cleaning review #407
Cleaning review #408
Cleaning review #409
Cleaning review #410
Cleaning review #411
Cleaning review #412
Cleaning review #413
Cleaning review #414
Cleaning review #415
Cleaning review #416
Cleaning review #417
Cleaning review #418
Cleaning review #419
Cleaning review #420
Cleaning review #421
Cleaning review #422
Cleaning review #423
Cleaning review #424
Cleaning review #425
Cleaning review #426
Cleaning review #427
Cleaning review #428
Cleaning review #429
Cleaning review #430
Cleaning review #431
Cleaning review #432
Cleaning review #433
Cleaning review #434
Cleaning review #435
Cleaning review #436
Cleaning review #437
Cleaning review #438
Cleaning review #439
Cleaning review #440
Cleaning review #441
Cleaning review #442
Cleaning review #443
Cleaning review #444
Cleaning review #445
Cleaning review #446
Cleaning review #447
Cleaning review #448
Cleaning review #449
Cleaning review #450
Cleaning review #451
Cleaning review #452
Cleaning review #453
Cleaning review #454
Cleaning review #455
Cleaning review #456
Cleaning review #457
Cleaning review #458
Cleaning review #459
Cleaning review #460
Cleaning review #461
Cleaning review #462
Cleaning review #463
Cleaning review #464
Cleaning review #465
Cleaning review #466
Cleaning review #467
Cleaning review #468
Cleaning review #469
Cleaning review #470
Cleaning review #471
Cleaning review #472
Cleaning review #473
Cleaning review #474
Cleaning review #475
Cleaning review #476
Cleaning review #477
Cleaning review #478
Cleaning review #479
Cleaning review #480
Cleaning review #481
Cleaning review #482
Cleaning review #483
Cleaning review #484
Cleaning review #485
Cleaning review #486
Cleaning review #487
Cleaning review #488
Cleaning review #489
Cleaning review #490
Cleaning review #491
Cleaning review #492
Cleaning review #493
Cleaning review #494
Cleaning review #495
Cleaning review #496
Cleaning review #497
Cleaning review #498
Cleaning review #499
Cleaning review #500
Total Number of Movie Review Document Vectors:  500

Creating Corpus of Combined Reviews of Best and Worst Horror Movies

In [30]:
all_reviews = all_good_movie_reviews_500 + all_bad_movie_reviews_500
In [31]:
all_reviews_text = []
all_reviews_ttl = []
for i in range(len(all_reviews)):
    all_reviews_text.append(all_reviews[i][0])
    all_reviews_ttl.append(all_reviews[i][1][0])
In [32]:
reg_ex = regex.compile('[^a-zA-Z]')
In [33]:
dictionary = corpora.Dictionary(all_reviews_text)
In [34]:
#remove extremes (similar to the min/max df step used when creating the tf-idf matrix)
#filtered_dict = dictionary.filter_extremes(no_below=1, no_above=0.8)

#convert the dictionary to a bag of words corpus for reference
corpus = [dictionary.doc2bow(text) for text in all_reviews_text]
In [35]:
all_reviews_joined = [' '.join(w) for w in all_reviews_text]
In [41]:
mod_list_lda_umass, coh_list_lda_umass = compute_coherence_values(LdaModel, dictionary, all_reviews_text, max_topics=20, 
                                                                     corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                                                     min_topics=2, stride=3, n_top_words=False, measure='u_mass')

mod_list_lda_cv, coh_list_lda_cv = compute_coherence_values(LdaModel, dictionary, all_reviews_text, max_topics=20, 
                                                                     corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                                                     min_topics=2, stride=3, n_top_words=False, measure='c_v')
Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'gensim.models.ldamodel.LdaModel'> using the U_MASS measure


Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'gensim.models.ldamodel.LdaModel'> using the C_V measure


In [42]:
best_num_lda_topics = show_best_num_topics('LDA', coh_list_lda_umass, coh_list_lda_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of LDA topics to use is: {}'.format(best_num_lda_topics))
The most coherent number of LDA topics using the U_MASS measure is: 2
The most coherent number of LDA topics using the C_V measure is: 8


The most coherent number of LDA topics to use is: 8
In [43]:
lda_mod = models.LdaModel(corpus, num_topics=best_num_lda_topics,  
                                id2word=dictionary, 
                                update_every=0, #batch training
                                chunksize=1000, 
                                passes=500,
                                random_state=121)
In [44]:
#Perplexity measure (lower is better)
print('The Perplexity measure of the final LDA model is: {}'.format(lda_mod.log_perplexity(corpus)))
The Perplexity measure of the final LDA model is: -8.298304571058113
In [39]:
#topics_matrix = lda_mod.show_topics(formatted=True, num_words=20)
#topics_matrix = np.array(topics_matrix)
#topics_matrix.shape
In [40]:
#topic_words = topics_matrix[:,1]
#for i in topic_words:
#    print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
#    print('\n')
In [45]:
lda_df = get_topics(lda_mod, best_num_lda_topics)
#print(lda_df)
lda_df
Out[45]:
Topic # 01 Topic # 02 Topic # 03 Topic # 04 Topic # 05 Topic # 06 Topic # 07 Topic # 08
0 watch story story character story story story story
1 story time dont story watch end watch character
2 think scene character people time character scene time
3 time think time time think time time feel
4 feel character act work original plot love dont
5 character end work dont feel start people scene
6 scare feel people scene people scene original people
7 people room feel love scene watch ghost think
8 family leave audience place play fan dont look
9 leave plot watch play end director feel watch
10 end people look set plot ride doesnt doesnt
11 look look home real work turn think work
12 ive watch scene soon love dead creepy scare
13 want vampire genre life look think ive director
14 work play scary feel director genre character dark
15 sarah sense end plot dont people remake right
16 act ive love violence new work end play
17 original 1408 director dark girl pretty zombie scary
18 juno want scare want want enjoy want house
19 build kind think start character look act end
In [46]:
lda_hellinger_topic_distances = get_most_similar_topics(lda_mod, lda_df, num_topics=best_num_lda_topics)
pretty_print(lda_hellinger_topic_distances)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.4837110411248617, 'Topic # 01', 'Topic # 06')
 (0.48174251539487667, 'Topic # 03', 'Topic # 06')
 (0.4748510044883907, 'Topic # 06', 'Topic # 07')
 (0.4717990853726296, 'Topic # 04', 'Topic # 06')
 (0.46271629711555196, 'Topic # 03', 'Topic # 07')
 (0.46173491768957614, 'Topic # 02', 'Topic # 03')
 (0.46127626256335685, 'Topic # 02', 'Topic # 06')
 (0.46001700327696304, 'Topic # 03', 'Topic # 04')
 (0.4575400998567088, 'Topic # 02', 'Topic # 07')
 (0.4549526255283364, 'Topic # 01', 'Topic # 04')
 (0.45447676874309, 'Topic # 02', 'Topic # 04')
 (0.4544377282672066, 'Topic # 01', 'Topic # 02')
 (0.4515640911141052, 'Topic # 06', 'Topic # 08')
 (0.45126237662510776, 'Topic # 04', 'Topic # 07')
 (0.44308567455525505, 'Topic # 04', 'Topic # 05')
 (0.4390830521079113, 'Topic # 01', 'Topic # 05')
 (0.4389738920982188, 'Topic # 01', 'Topic # 07')
 (0.4387911167272729, 'Topic # 04', 'Topic # 08')
 (0.43786335685161076, 'Topic # 05', 'Topic # 06')
 (0.4355284852956487, 'Topic # 02', 'Topic # 05')
 (0.43441019971552597, 'Topic # 03', 'Topic # 05')
 (0.4333198334330377, 'Topic # 05', 'Topic # 07')
 (0.42670160586444283, 'Topic # 01', 'Topic # 03')
 (0.4234971666361381, 'Topic # 01', 'Topic # 08')
 (0.4183075835478236, 'Topic # 07', 'Topic # 08')
 (0.4160318363128511, 'Topic # 03', 'Topic # 08')
 (0.40846795466068725, 'Topic # 02', 'Topic # 08')
 (0.39419233251966707, 'Topic # 05', 'Topic # 08')

______________________________________________________________________
In [47]:
lda_mod_dom_topics_df, lda_mod_dom_topics_final = get_dominant_topics(lda_mod, corpus, all_reviews_joined)
In [48]:
#lda_mod_dom_topics_final.style.applymap(color_green).applymap(make_bold)
lda_mod_dom_topics_final_first30 = lda_mod_dom_topics_final.head(30)
lda_mod_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Out[48]:
Dominant_Topic Probability_Contribution Topic_Keywords Original_Text
Document_Number
0 4 0.9921 story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times
1 2 0.9883 story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let
2 6 0.9972 story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute
3 7 0.988 story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole
4 7 0.6144 story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems
5 3 0.9961 character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue
6 3 0.9922 character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome
7 1 0.9856 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended
8 4 0.993 story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong
9 7 0.9971 story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate
10 7 0.6891 story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+
11 1 0.9949 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder
12 3 0.9541 character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty
13 1 0.9956 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original
14 1 0.9786 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch
15 6 0.965 story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played
16 6 0.9663 story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat
17 4 0.9687 story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader
18 1 0.9757 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak
19 4 0.927 story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character zombie plenty thistle cheerleader splendid gods gift debut story peter end
20 0 0.9972 watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring
21 5 0.4848 story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint
22 2 0.5496 story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced
23 3 0.9964 character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house
24 0 0.5532 watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time
25 6 0.992 story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction
26 5 0.9925 story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer
27 5 0.99 story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black
28 5 0.9952 story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point
29 1 0.9763 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
In [49]:
lda_most_representative_docs = get_most_representative_docs(lda_mod_dom_topics_final, n_topics=20)
lda_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[49]:
Probability_Contribution Topic_Keywords Most_Representative_Document
Topic_Number
0 0.3801 watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build night wife expect huge fan old school nightmare elm street blood gore dead alive include foreign devil prepare minute heart pump suspense feel kid watch scary scare acting atmosphere creepy face look alive blink moved couldnt sleep night year old year old wow watch caution
1 0.5295 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind disgust spend hour raw thats disgust grotesque intrigue canst look act character interest bite time pick scene pointless haze rituals eye glue screen theme human instinct terrify horrid entertain main character turn normal girl different people critique end took love overall definitely worth watch masterpiece break new ground mess disgust entertain runtime
2 0.5266 story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think trick treat night brightest leicester square need round applause end doesnt usually happen uk wasnt fact dougherty thousand tar t undoubtedly seen moment start feel tell fair money chuck set look fantastic straight dvd october theatrical run 2007 unbelievable act effect direction masterful basically set halloweens bunch story interweave clever sort creepshow story isnt standalone time end didnt think bundle surprise litter wonder prom night remake countless saw theatrical run awesome shelve disregarded true masterpiece admission instant classic
3 0.6445 character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start having days think prepare near begin scene zero psycho second flat begin 2004 dawn dead wildly chaotic kick-off scene unlike laugh chomp popcorn laugh matter theres violence theres fear tension on-screen violence absolute shock scene scene ordinary people place impossible situation escape time course implacable predator hunt rage virus military attempt maintain control outbreak visit unspeakable innocent people control scale virus severe plan military implement completely plausible action scene masterfully effectively place viewer point view victim crazed scarily human zombies portrayal violence pull punches people age group walk life destroy remorse attempt soft-pedal fragility human life earth vulnerability right nasty virus thought stay youve leave theater add nice taste fear soundtracks amaze convey tension dread sadness scenes story fairly tight complaint act soldiers didnt feel authentic reason overall id zombie ive seen fact effective thriller ive seen
4 0.5072 story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character nattevagten thriller edge seat saw right begin original story suck doesnt let end thrillers grip extremely rare time people guess twist saw james wan leigh whannell creative head project set new standards think youre hard-boiled think watch saw creep surprise expectations saw advertise new se7en definitely traditions saw job creepy jigsaw gruesome killer cinema looking time wan channel monster peer drift ridiculousness establish villain superhuman evil saw near trap cop depict stupid killer unrealistically smart outshine opponent perfect plans hey se7en silence lambs didnt care realism saw flaws sped-up track shot time structure script weird jump period time characters line bite clichéd consider day independent filmmakers literally budget inappropriate petty technical subtleties wan channel original stir canst remember time ive surprise movies final twist saw end didnt thriller original piece independent film-making cube ism look forward wan whannell career develop fine sleeper
5 0.4848 story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint
6 0.5212 story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act shutters begin thun young photographer girlfriend jane accidentally run young woman drive home decide leave dead victim drive thun discover strange mysterious shadow appear photo think thats bad picture realize sinister shadow-both tun jane plague extremely unsettle dreams unable cope start investigate phenomenon ghost spirit photography lead discovery tunes past possible clue identity ghostly nemesis shutter hit thailand easy notice whit offer truly eerie moment share book scare tooth final image scream act surprisingly fascinate watched look
7 0.4681 story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end herems theater sport gorgeous black white photography farsis vampire art-house flick apparently unstoppable hype machine power vice kino lorber usually distributor worth watching promote picture actual appeal dishonestly pace trailer ingredient sound amazing perfect bait theres market hallmark different turn mean yes farsis culturally void iranian bakersfield california bakersfield starkly cinematics endeavor feel extremely western youre catch glimpse under- represent culture youre cool role farsi speak american actors mean serve place inside persian community bakersfield interest thats cases fictional western-pulp style iran community bad town set series creative decision cool factor deep purpose yes art house flick ambitious kind celebrate style substance assume disaffection equal cool choose aesthetic principle theme turn stab greatness imitate giant nouvelle vague post punk wave cinema cite spaghetti westerns influence thats soundtracks image music unsupported character theme miss place yes vampire lead affect beautiful cool vampire narrative soon jarmusch lovers left alive jarmusch season filmmaker able non-narrative attempts place emphasis style character theme lovers left alive lovers music video tonal exercise girl walks home night isn t doesnt aspire scary creepy obviously doesnt ism fine tone dont expect day scary vampire movieland leave gorgeous photography lyle vincent frankly save flick intolerable bear feature strong cinematic moments shoot truly joy rely completely love beautiful cinematography thats save grace bright spots music excellent rise height sequence suppose supporting unquestionably director use music affect nice moment sprinkle final climactic death beautifully piece filmmaking lead lovely fun watch scene end tedious meditative pretty obvious short stretch means love cinema slow contemplate languid shoot real hold underneath image music feel mean exercise ism fresh different meaning surface barely qualifies theres new unique tired snoop amirpour future hope aesthetic day engine purpose cinema unique
In [50]:
lda_topic_dist = get_topic_distribution(lda_mod_dom_topics_final, lda_most_representative_docs, n_topics=20)
lda_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[50]:
Documents_per_Topic Percent_of_Total_Corpus Topic_Keywords
Topic_Number
0 140 0.14 watch, story, think, time, feel, character, scare, people, family, leave, end, look, ive, want, work, sarah, act, original, juno, build
1 128 0.128 story, time, scene, think, character, end, feel, room, leave, plot, people, look, watch, vampire, play, sense, ive, 1408, want, kind
2 112 0.112 story, dont, character, time, act, work, people, feel, audience, watch, look, home, scene, genre, scary, end, love, director, scare, think
3 64 0.064 character, story, people, time, work, dont, scene, love, place, play, set, real, soon, life, feel, plot, violence, dark, want, start
4 160 0.16 story, watch, time, think, original, feel, people, scene, play, end, plot, work, love, look, director, dont, new, girl, want, character
5 92 0.092 story, end, character, time, plot, start, scene, watch, fan, director, ride, turn, dead, think, genre, people, work, pretty, enjoy, look
6 120 0.12 story, watch, scene, time, love, people, original, ghost, dont, feel, doesnt, think, creepy, ive, character, remake, end, zombie, want, act
7 184 0.184 story, character, time, feel, dont, scene, people, think, look, watch, doesnt, work, scare, director, dark, right, play, scary, house, end
In [51]:
prepared = pyLDAvis.gensim.prepare(lda_mod, corpus, dictionary)
display(pyLDAvis.display(prepared))
In [52]:
#Cf. https://github.com/RaRe-Technologies/gensim/issues/2137 (for fix)

#File dl: http://mallet.cs.umass.edu/dist/mallet-2.0.8.zip
os.environ['MALLET_HOME'] = '.\\mallet-2.0.8\\'
mallet_path = '.\\mallet-2.0.8\\bin\\mallet'
In [53]:
mod_list_ldamallet_umass, coh_list_ldamallet_umass = compute_coherence_values(LdaMallet, dictionary, all_reviews_text, max_topics=20, 
                                                                     corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                                                     min_topics=2, stride=3, n_top_words=False, measure='u_mass', 
                                                                     mallet_flag=True, mallet_path=mallet_path)

mod_list_ldamallet_cv, coh_list_ldamallet_cv = compute_coherence_values(LdaMallet, dictionary, all_reviews_text, max_topics=20, 
                                                                     corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                                                     min_topics=2, stride=3, n_top_words=False, measure='c_v', 
                                                                     mallet_flag=True, mallet_path=mallet_path)
Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'gensim.models.wrappers.ldamallet.LdaMallet'> using the U_MASS measure


Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'gensim.models.wrappers.ldamallet.LdaMallet'> using the C_V measure


In [54]:
best_num_ldamallet_topics = show_best_num_topics('LDA_Mallet', coh_list_ldamallet_umass, coh_list_ldamallet_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of LDA_Mallet topics to use is: {}'.format(best_num_ldamallet_topics))
The most coherent number of LDA_Mallet topics using the U_MASS measure is: 2
The most coherent number of LDA_Mallet topics using the C_V measure is: 8


The most coherent number of LDA_Mallet topics to use is: 8
In [55]:
lda_mallet = LdaMallet(mallet_path, corpus=corpus, num_topics=best_num_ldamallet_topics, id2word=dictionary)
In [60]:
#topics_matrix = lda_mallet.show_topics(formatted=True, num_words=20)
#topics_matrix = np.array(topics_matrix)
#topics_matrix.shape
In [61]:
#topic_words = topics_matrix[:,1]
#for i in topic_words:
#    print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
#    print('\n')
In [56]:
lda_mallet_df = get_topics(lda_mallet, best_num_ldamallet_topics)
#print(lda_mallet_df)
lda_mallet_df
Out[56]:
Topic # 01 Topic # 02 Topic # 03 Topic # 04 Topic # 05 Topic # 06 Topic # 07 Topic # 08
0 story house scene watch scene performance character time
1 ghost plot dark original vampire actor people feel
2 love scare story fan shoot creepy dont leave
3 end story work ism girl story sound doesnt
4 scary genre ring dont work room live character
5 play family psychological youre days years thriller story
6 short audience twist remake night release kill dont
7 world haunt excellent minute town time love watch
8 time true water flick woman year killer sense
9 start add tale ive nature night play start
10 creepy doesnt performance dead guy young end happen
11 work act fact people plot genre feel end
12 suspense feel slowly enjoy life director situation didnt
13 bite music american bad zombie play place friend
14 funny sinister japanese movies home stand time real
15 moment day brilliant classic experience treat world understand
16 hollywood conjuring revenge time wife fan school scary
17 atmosphere eye viewer wasnt shadow set problem people
18 genre element perfect love real title young characters
19 canst character pretty expect follow cinematography high hard
In [57]:
ldamallet_hellinger_topic_distances = get_most_similar_topics(lda_mallet, lda_mallet_df, num_topics=best_num_ldamallet_topics, mallet_flag=True)
pretty_print(ldamallet_hellinger_topic_distances)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.9545340440576985, 'Topic # 04', 'Topic # 05')
 (0.9426037977006119, 'Topic # 03', 'Topic # 04')
 (0.9359234608592298, 'Topic # 02', 'Topic # 07')
 (0.9341593554859184, 'Topic # 04', 'Topic # 06')
 (0.9340884820207142, 'Topic # 01', 'Topic # 04')
 (0.9316609666635765, 'Topic # 06', 'Topic # 08')
 (0.9258880611682975, 'Topic # 05', 'Topic # 06')
 (0.9247568336998111, 'Topic # 03', 'Topic # 07')
 (0.924315383410101, 'Topic # 05', 'Topic # 08')
 (0.9237283410570338, 'Topic # 04', 'Topic # 07')
 (0.9218963025154837, 'Topic # 02', 'Topic # 05')
 (0.9215309384100554, 'Topic # 06', 'Topic # 07')
 (0.9189364773675167, 'Topic # 04', 'Topic # 08')
 (0.9178189761080324, 'Topic # 02', 'Topic # 04')
 (0.9145282089633445, 'Topic # 03', 'Topic # 05')
 (0.9108689923818001, 'Topic # 05', 'Topic # 07')
 (0.9091161059483536, 'Topic # 01', 'Topic # 03')
 (0.9087474646722671, 'Topic # 01', 'Topic # 05')
 (0.9087000354313345, 'Topic # 01', 'Topic # 07')
 (0.907448147217302, 'Topic # 03', 'Topic # 06')
 (0.9056529464165876, 'Topic # 01', 'Topic # 06')
 (0.9049301973905479, 'Topic # 02', 'Topic # 03')
 (0.9039229616814698, 'Topic # 02', 'Topic # 06')
 (0.900690730724466, 'Topic # 01', 'Topic # 02')
 (0.8923821813919287, 'Topic # 07', 'Topic # 08')
 (0.8868980857601233, 'Topic # 01', 'Topic # 08')
 (0.8863877592324878, 'Topic # 03', 'Topic # 08')
 (0.8783005976699784, 'Topic # 02', 'Topic # 08')

______________________________________________________________________
In [58]:
lda_mallet_dom_topics_df, lda_mallet_dom_topics_final = get_dominant_topics(lda_mallet, corpus, all_reviews_joined)
In [59]:
#lda_mallet_dom_topics_final.style.applymap(color_green).applymap(make_bold)
lda_mallet_dom_topics_final_first30 = lda_mallet_dom_topics_final.head(30)
lda_mallet_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Out[59]:
Dominant_Topic Probability_Contribution Topic_Keywords Original_Text
Document_Number
0 1 0.2769 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times
1 2 0.2751 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let
2 7 0.4288 time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute
3 7 0.1803 time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole
4 1 0.2393 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems
5 4 0.3068 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue
6 4 0.3941 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome
7 3 0.1579 watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended
8 4 0.4505 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong
9 4 0.4439 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate
10 2 0.3339 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+
11 2 0.4811 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder
12 2 0.2358 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty
13 2 0.489 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original
14 3 0.1925 watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch
15 3 0.155 watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played
16 4 0.1739 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat
17 5 0.1556 performance, actor, creepy, story, room, years, release, time, year, night, young, genre, director, play, stand, treat, fan, set, title, cinematography train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader
18 3 0.2029 watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak
19 0 0.1371 story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst zombie plenty thistle cheerleader splendid gods gift debut story peter end
20 1 0.6055 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring
21 1 0.3957 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint
22 1 0.2305 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced
23 1 0.4168 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house
24 1 0.3826 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time
25 2 0.3585 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction
26 2 0.2874 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer
27 0 0.1951 story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black
28 2 0.3035 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point
29 2 0.1814 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
In [60]:
lda_mallet_most_representative_docs = get_most_representative_docs(lda_mallet_dom_topics_final, n_topics=20)
lda_mallet_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[60]:
Probability_Contribution Topic_Keywords Most_Representative_Document
Topic_Number
0 0.1352 story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst zombie plenty thistle cheerleader splendid gods gift debut story peter end
1 0.1492 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut
2 0.1617 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty unlike reviews ism cheap shot ju-on fan supernatural owe decide wholeheartedly agree uncomfortable watch experience powerful ring dark water exorcista scary scary stuff
3 0.1515 watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut
4 0.1709 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat
5 0.1399 performance, actor, creepy, story, room, years, release, time, year, night, young, genre, director, play, stand, treat, fan, set, title, cinematography fantastic performance start james mcevoy reason watch personality display distinct interest watch anya breath-taking witch fine job took night shyamalan reassert director follow string poor absolutely wait sequel conclusion unbreakable series arrive january
6 0.1538 character, people, dont, sound, live, thriller, kill, love, killer, play, end, feel, situation, place, time, world, school, problem, young, high ive read review ism die hard thriller fan think slow- bad ism build ending actress play eden think cast better ill worth watch thrillers
7 0.1423 time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard point mince words brad anderson session ive time intelligent well-written completely unpredictable look didnt notice view edit photography work soundtrack downright creepy recently manage lie awake night dario argentous opera tobe hoopers texas chain saw massacre list include honestly excuse folks doesnt
In [61]:
lda_mallet_topic_dist = get_topic_distribution(lda_mallet_dom_topics_final, lda_mallet_most_representative_docs, n_topics=20)
lda_mallet_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[61]:
Documents_per_Topic Percent_of_Total_Corpus Topic_Keywords
Topic_Number
0 120 0.12 story, ghost, love, end, scary, play, short, world, time, start, creepy, work, suspense, bite, funny, moment, hollywood, atmosphere, genre, canst
1 110 0.11 house, plot, scare, story, genre, family, audience, haunt, true, add, doesnt, act, feel, music, sinister, day, conjuring, eye, element, character
2 133 0.133 scene, dark, story, work, ring, psychological, twist, excellent, water, tale, performance, fact, slowly, american, japanese, brilliant, revenge, viewer, perfect, pretty
3 201 0.201 watch, original, fan, ism, dont, youre, remake, minute, flick, ive, dead, people, enjoy, bad, movies, classic, time, wasnt, love, expect
4 84 0.084 scene, vampire, shoot, girl, work, days, night, town, woman, nature, guy, plot, life, zombie, home, experience, wife, shadow, real, follow
5 96 0.096 performance, actor, creepy, story, room, years, release, time, year, night, young, genre, director, play, stand, treat, fan, set, title, cinematography
6 110 0.11 character, people, dont, sound, live, thriller, kill, love, killer, play, end, feel, situation, place, time, world, school, problem, young, high
7 146 0.146 time, feel, leave, doesnt, character, story, dont, watch, sense, start, happen, end, didnt, friend, real, understand, scary, people, characters, hard
In [68]:
#NOTE: .malletmodel2ldamodel() is buggy!
    #Cf. https://github.com/RaRe-Technologies/gensim/issues/1203
    #Cf. https://github.com/RaRe-Technologies/gensim/issues/2069
    
#For workaround, Cf. http://jeriwieringa.com/2018/07/17/pyLDAviz-and-Mallet/#comment-4018495276

#mallet_2_lda_mod = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(lda_mallet)
#prepared = pyLDAvis.gensim.prepare(mallet_2_lda_mod, corpus, dictionary)
#display(pyLDAvis.display(prepared))

LDA Mallet: Second Pass with second highest C_V Coherence Optimized number of topics

In [62]:
lda_mallet_2 = LdaMallet(mallet_path, corpus=corpus, num_topics=14, id2word=dictionary)
In [64]:
lda_mallet_df_2 = get_topics(lda_mallet_2, 14)
#print(lda_mallet_df_2)
lda_mallet_df_2
Out[64]:
Topic # 01 Topic # 02 Topic # 03 Topic # 04 Topic # 05 Topic # 06 Topic # 07 Topic # 08 Topic # 09 Topic # 10 Topic # 11 Topic # 12 Topic # 13 Topic # 14
0 style original dont scare work story scene people story sound house people zombie family
1 vampire watch character scary nature end story story ghost time didnt feel people youre
2 scene fan room ive woman play dark watch sinister love story friend shoot work
3 feel remake time jump wrong short ring scene love character feel character dead twist
4 cinematography classic thriller watch fall fun real turn director ism plot leave days expect
5 brilliant flick ive haunt describe time watch character scary dont girl real ride start
6 excellent dead story real director creepy sense viewer home live guy japanese attack add
7 ism treat past creepy mind director plot reason fan monster happen life scene end
8 music gore play time fear bite water blood cast hear kid world character dont
9 tale year work doesnt include start visual truth atmosphere pretty night act tension characters
10 line evil doesnt extremely case surprise time plot family face review shock villain time
11 set release 1408 vampire pain funny david feel war walk start time completely act
12 bad trick writer conjuring child pretty experience game child start break sarah point minute
13 genre enjoy things performance death suspense point idea supernatural leave leave juno action doesnt
14 place fans write genre characters bring face camera element follow enjoy high town light
15 beautiful performance twist james emotion minute resolution body screen world end kill woman strong
16 atmosphere genre ll theyre love night sound review orphanage kill sense happen virus pace
17 girl sequel john feel psychological gore realize raw house loud bad year infect build
18 absolutely teen read music hard clich weird isnt simple human money event person rest
19 kind fun director scares emotional fan mike work young eat run young begin genre
In [115]:
ldamallet_hellinger_topic_distances_2 = get_most_similar_topics(lda_mallet_2, lda_mallet_df_2, num_topics=14, mallet_flag=True)
pretty_print(ldamallet_hellinger_topic_distances_2)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.9640064905425804, 'Topic # 02', 'Topic # 07')
 (0.9630872911656057, 'Topic # 05', 'Topic # 11')
 (0.959551339144133, 'Topic # 04', 'Topic # 05')
 (0.9583074176732344, 'Topic # 09', 'Topic # 13')
 (0.9572472126294248, 'Topic # 02', 'Topic # 11')
 (0.9572410852906693, 'Topic # 06', 'Topic # 13')
 (0.9564785062656007, 'Topic # 09', 'Topic # 11')
 (0.9515582230378915, 'Topic # 04', 'Topic # 10')
 (0.9512168721624514, 'Topic # 04', 'Topic # 13')
 (0.9502617007912305, 'Topic # 09', 'Topic # 10')
 (0.9499244066667502, 'Topic # 02', 'Topic # 05')
 (0.9468286136008315, 'Topic # 02', 'Topic # 13')
 (0.9430261325783785, 'Topic # 05', 'Topic # 06')
 (0.9425823285725585, 'Topic # 06', 'Topic # 12')
 (0.9420040275533847, 'Topic # 01', 'Topic # 05')
 (0.9417975252917109, 'Topic # 02', 'Topic # 03')
 (0.9402472926059382, 'Topic # 02', 'Topic # 08')
 (0.9402072660218292, 'Topic # 13', 'Topic # 14')
 (0.9397780668554646, 'Topic # 02', 'Topic # 10')
 (0.9380780343337605, 'Topic # 02', 'Topic # 12')
 (0.9380136294785817, 'Topic # 02', 'Topic # 06')
 (0.936734608528866, 'Topic # 01', 'Topic # 03')
 (0.9366856750100395, 'Topic # 01', 'Topic # 02')
 (0.9366577936877349, 'Topic # 05', 'Topic # 09')
 (0.9364644951443643, 'Topic # 01', 'Topic # 13')
 (0.9363631057812704, 'Topic # 08', 'Topic # 10')
 (0.9362873858656198, 'Topic # 03', 'Topic # 05')
 (0.9360853350982958, 'Topic # 04', 'Topic # 11')
 (0.9357046974338342, 'Topic # 07', 'Topic # 11')
 (0.9352819159927208, 'Topic # 05', 'Topic # 07')
 (0.934566993652253, 'Topic # 02', 'Topic # 14')
 (0.9343355961517718, 'Topic # 01', 'Topic # 06')
 (0.9342267700044657, 'Topic # 11', 'Topic # 13')
 (0.9339365932274444, 'Topic # 01', 'Topic # 10')
 (0.9326864835629602, 'Topic # 05', 'Topic # 13')
 (0.9324200074954956, 'Topic # 06', 'Topic # 07')
 (0.9320295173396599, 'Topic # 10', 'Topic # 13')
 (0.9310044895645258, 'Topic # 04', 'Topic # 12')
 (0.9310004677761589, 'Topic # 06', 'Topic # 10')
 (0.9307135461378772, 'Topic # 08', 'Topic # 09')
 (0.9305707470324849, 'Topic # 04', 'Topic # 08')
 (0.9299139984274493, 'Topic # 06', 'Topic # 08')
 (0.928261964297447, 'Topic # 01', 'Topic # 09')
 (0.9280944444529539, 'Topic # 09', 'Topic # 12')
 (0.9277725014527576, 'Topic # 07', 'Topic # 10')
 (0.9276461452648772, 'Topic # 07', 'Topic # 13')
 (0.9269518510732289, 'Topic # 08', 'Topic # 14')
 (0.9268323096340466, 'Topic # 01', 'Topic # 07')
 (0.9264987433284755, 'Topic # 02', 'Topic # 09')
 (0.9262790258347953, 'Topic # 07', 'Topic # 09')
 (0.9255696046789911, 'Topic # 03', 'Topic # 13')
 (0.9253181565349187, 'Topic # 01', 'Topic # 14')
 (0.9250726011887772, 'Topic # 01', 'Topic # 11')
 (0.9248987448461002, 'Topic # 04', 'Topic # 09')
 (0.924804376326733, 'Topic # 10', 'Topic # 11')
 (0.9241945250457422, 'Topic # 05', 'Topic # 14')
 (0.9236020907564638, 'Topic # 05', 'Topic # 08')
 (0.923549657003945, 'Topic # 06', 'Topic # 11')
 (0.9227998720545468, 'Topic # 04', 'Topic # 07')
 (0.9225210323813859, 'Topic # 05', 'Topic # 10')
 (0.9210296589008242, 'Topic # 02', 'Topic # 04')
 (0.9205754795844071, 'Topic # 07', 'Topic # 14')
 (0.918901528198737, 'Topic # 05', 'Topic # 12')
 (0.9188051769175941, 'Topic # 01', 'Topic # 08')
 (0.9182552888186247, 'Topic # 09', 'Topic # 14')
 (0.9178322793105529, 'Topic # 08', 'Topic # 11')
 (0.9173822837777433, 'Topic # 04', 'Topic # 14')
 (0.9166991032071017, 'Topic # 11', 'Topic # 14')
 (0.9160713848527243, 'Topic # 03', 'Topic # 10')
 (0.9156962901359114, 'Topic # 06', 'Topic # 09')
 (0.9154662018446263, 'Topic # 10', 'Topic # 14')
 (0.9150848628709455, 'Topic # 03', 'Topic # 07')
 (0.9139783821894085, 'Topic # 03', 'Topic # 08')
 (0.9139618472217662, 'Topic # 12', 'Topic # 14')
 (0.9124574201528883, 'Topic # 01', 'Topic # 04')
 (0.9118831477143167, 'Topic # 03', 'Topic # 06')
 (0.910369561902317, 'Topic # 01', 'Topic # 12')
 (0.9090191200018434, 'Topic # 04', 'Topic # 06')
 (0.9085294755991589, 'Topic # 07', 'Topic # 08')
 (0.9076584663540122, 'Topic # 08', 'Topic # 13')
 (0.9068081504990716, 'Topic # 06', 'Topic # 14')
 (0.9060864158201576, 'Topic # 03', 'Topic # 09')
 (0.9053152568445154, 'Topic # 07', 'Topic # 12')
 (0.904457235584954, 'Topic # 03', 'Topic # 12')
 (0.9020310763313824, 'Topic # 12', 'Topic # 13')
 (0.8977620593410172, 'Topic # 11', 'Topic # 12')
 (0.8957287994479677, 'Topic # 10', 'Topic # 12')
 (0.8928704047386233, 'Topic # 03', 'Topic # 14')
 (0.8902855908351145, 'Topic # 03', 'Topic # 11')
 (0.8834213857263032, 'Topic # 03', 'Topic # 04')
 (0.8798037125279303, 'Topic # 08', 'Topic # 12')

______________________________________________________________________
In [66]:
lda_mallet_dom_topics_df_2, lda_mallet_dom_topics_final_2 = get_dominant_topics(lda_mallet_2, corpus, all_reviews_joined)
In [67]:
#lda_mallet_dom_topics_final_2.style.applymap(color_green).applymap(make_bold)
lda_mallet_dom_topics_final_first30_2 = lda_mallet_dom_topics_final_2.head(30)
lda_mallet_dom_topics_final_first30_2.style.applymap(color_green).applymap(make_bold)
Out[67]:
Dominant_Topic Probability_Contribution Topic_Keywords Original_Text
Document_Number
0 6 0.449 scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times
1 6 0.1745 scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let
2 6 0.2982 scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute
3 0 0.1639 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole
4 10 0.1271 house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems
5 12 0.3841 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue
6 12 0.3705 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome
7 2 0.1472 dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended
8 12 0.3971 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong
9 12 0.4656 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate
10 0 0.3686 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+
11 0 0.2909 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder
12 0 0.1836 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty
13 0 0.4459 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original
14 1 0.1126 original, watch, fan, remake, classic, flick, dead, treat, gore, year, evil, release, trick, enjoy, fans, performance, genre, sequel, teen, fun fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch
15 10 0.1489 house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played
16 4 0.125 work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat
17 4 0.1189 work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader
18 9 0.1322 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak
19 2 0.0859 dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director zombie plenty thistle cheerleader splendid gods gift debut story peter end
20 3 0.4558 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring
21 3 0.3445 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint
22 3 0.2398 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced
23 3 0.3782 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house
24 3 0.3111 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time
25 9 0.265 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction
26 9 0.36 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer
27 9 0.3351 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black
28 9 0.4401 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point
29 12 0.0948 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
In [68]:
lda_mallet_most_representative_docs_2 = get_most_representative_docs(lda_mallet_dom_topics_final_2, n_topics=20)
lda_mallet_most_representative_docs_2.style.applymap(color_green).applymap(make_bold)
Out[68]:
Probability_Contribution Topic_Keywords Most_Representative_Document
Topic_Number
0 0.11 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind think true original break sweat certain points physical effect audience astonish moment scene transition literally breathtaking didnt believe director certain solely shock people suggested look lush style stun compositions favourite story final sickly grandson original character source hukkle want director interest time linger mind
1 0.1054 original, watch, fan, remake, classic, flick, dead, treat, gore, year, evil, release, trick, enjoy, fans, performance, genre, sequel, teen, fun create extremely tense atmosphere moment starts plenty gore violence bombard eyes mention twist watch night light expect freak think typical instead completely intrigue story line utterly draw 100 minutes end feel scared feel offer satisfaction leave slightly disturb thoughtful state nightmares plague day-dreams day thought stray far thats powerful want cringingly gory violent watch texas chainsaw massacre hostel want nightmarishly scary watch grudge ring want clever disturb excellent story twist watch saw
2 0.0859 dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director zombie plenty thistle cheerleader splendid gods gift debut story peter end
3 0.1174 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares unlike reviews ism cheap shot ju-on fan supernatural owe decide wholeheartedly agree uncomfortable watch experience powerful ring dark water exorcista scary scary stuff
4 0.1067 work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut
5 0.0836 story, end, play, short, fun, time, creepy, director, bite, start, surprise, funny, pretty, suspense, bring, minute, night, gore, clich, fan plot summary describe story scary funny
6 0.1194 scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike ive member idb year rarely time comment addition watch average 10-15 months split genre include lately thought ive disenfranchise proliferation shock hostel late saw captivity enter mist leave theater movies frank darabont adapt steven king novel short stories period bring human balance thats miss days unbelievable maybe ludicrous situation events character believable peel layer expose fear prejudice vulnerability foundation effective absolutely grip entire all-too-rare-these-days sense dread permeate scene leave emotionally exhaust end speak ending isnt talk ism opinion love split camp loved didn t conclusion entire storyline human condition wouldnt change things
7 0.1102 people, story, watch, scene, turn, character, viewer, reason, blood, truth, plot, feel, game, idea, camera, body, review, raw, isnt, work usually comment read peoples comments finish watch rec feel comment heart rate let hell terrify think dont repeat synopsis people story doesnt original zombie claustrophobic set totally different brilliant let place apartment building limit space person shoot add claustrophobic atmosphere scare wont mile hollywood movies shocking ending flick seen literally believe hollywood watch learn dont need gallon blood body fly scare live hell
8 0.1157 story, ghost, sinister, love, director, scary, home, fan, cast, atmosphere, family, war, child, supernatural, element, screen, orphanage, house, simple, young idea junks noise random jolts plenty fear deliver quietly compactly fuss suspenseful ive ring think better stomach knot prep chills rise wave calm water think bring robert dickman story near director uncanny skill place camera hold shot lead actress wonderful performance face elevator conclusion foreseeable--maybe end ghost story foreseeable--but nonetheless satisfying tale quietly disturb dread
9 0.1078 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat soon plot tranquil set family soon silence shatter event unit fall apart normal usual nowadays marvel fantasy disney release week usual helen murren samuel jackson jump consequence violence blood sexiness zaniness rest break ordinary case masterful let freely admit man power legs breasts hair hike skirt sex excited cold fish cut
10 0.1017 house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
11 0.1225 people, feel, friend, character, leave, real, japanese, life, world, act, shock, time, sarah, juno, high, kill, happen, year, event, young open scene guess familiar book said dont think possible satisfy adaption game thrones time need dwell novels create feel book tell story tell paper guess ill forgive filmmakers kid okay theyve manage portray loser relationship decently enough- consider run time time need development happen kid cut run time theyve change set 1958 bad choice changes theyre create scene arendt book theyre change problem execution constant jump scares cgi complete lack understand fact waiting scare scary throw constantly isnt terrifying alli didnt bored thought poor performance scary okay isnt adaptations
12 0.0948 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
13 0.1074 family, youre, work, twist, expect, start, add, end, dont, characters, time, act, minute, doesnt, light, strong, pace, build, rest, genre scary want genuinely jump time watch twice mute sound laugh write trust scene challenge old tickers cardboard box hallway mind means use footage integrate overall extremely disturbing definite potential sequel story set new chapter id watch day time highly recommend jump-scare fans
In [69]:
lda_mallet_topic_dist_2 = get_topic_distribution(lda_mallet_dom_topics_final_2, lda_mallet_most_representative_docs_2, n_topics=20)
lda_mallet_topic_dist_2.style.applymap(color_green).applymap(make_bold)
Out[69]:
Documents_per_Topic Percent_of_Total_Corpus Topic_Keywords
Topic_Number
0 79 0.079 style, vampire, scene, feel, cinematography, brilliant, excellent, ism, music, tale, line, set, bad, genre, place, beautiful, atmosphere, girl, absolutely, kind
1 100 0.1 original, watch, fan, remake, classic, flick, dead, treat, gore, year, evil, release, trick, enjoy, fans, performance, genre, sequel, teen, fun
2 92 0.092 dont, character, room, time, thriller, ive, story, past, play, work, doesnt, 1408, writer, things, write, twist, ll, john, read, director
3 94 0.094 scare, scary, ive, jump, watch, haunt, real, creepy, time, doesnt, extremely, vampire, conjuring, performance, genre, james, theyre, feel, music, scares
4 36 0.036 work, nature, woman, wrong, fall, describe, director, mind, fear, include, case, pain, child, death, characters, emotion, love, psychological, hard, emotional
5 84 0.084 story, end, play, short, fun, time, creepy, director, bite, start, surprise, funny, pretty, suspense, bring, minute, night, gore, clich, fan
6 63 0.063 scene, story, dark, ring, real, watch, sense, plot, water, visual, time, david, experience, point, face, resolution, sound, realize, weird, mike
7 65 0.065 people, story, watch, scene, turn, character, viewer, reason, blood, truth, plot, feel, game, idea, camera, body, review, raw, isnt, work
8 63 0.063 story, ghost, sinister, love, director, scary, home, fan, cast, atmosphere, family, war, child, supernatural, element, screen, orphanage, house, simple, young
9 60 0.06 sound, time, love, character, ism, dont, live, monster, hear, pretty, face, walk, start, leave, follow, world, kill, loud, human, eat
10 78 0.078 house, didnt, story, feel, plot, girl, guy, happen, kid, night, review, start, break, leave, enjoy, end, sense, bad, money, run
11 57 0.057 people, feel, friend, character, leave, real, japanese, life, world, act, shock, time, sarah, juno, high, kill, happen, year, event, young
12 59 0.059 zombie, people, shoot, dead, days, ride, attack, scene, character, tension, villain, completely, point, action, town, woman, virus, infect, person, begin
13 70 0.07 family, youre, work, twist, expect, start, add, end, dont, characters, time, act, minute, doesnt, light, strong, pace, build, rest, genre

HDP (Hierarchical Dirichlet Process)

Cf. http://proceedings.mlr.press/v15/wang11a/wang11a.pdf

In [70]:
hdp = models.HdpModel(corpus, id2word=dictionary, chunksize=1000, random_state=222)
In [100]:
#hdp_topics_matrix = hdp.show_topics(formatted=True, num_words=20)
In [101]:
#hdp_topics_matrix = np.array(hdp_topics_matrix)

#hdp_topics_matrix.shape
In [102]:
#hdp_topic_words = hdp_topics_matrix[:,1]
#for i in hdp_topic_words:
#    print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
#    print('\n')
In [116]:
hdp_df = get_topics(hdp, 20)
#print(hdp_df)
hdp_df
Out[116]:
Topic # 01 Topic # 02 Topic # 03 Topic # 04 Topic # 05 Topic # 06 Topic # 07 Topic # 08 Topic # 09 Topic # 10 Topic # 11 Topic # 12 Topic # 13 Topic # 14 Topic # 15 Topic # 16 Topic # 17 Topic # 18 Topic # 19 Topic # 20
0 room story scare prof park water character salinsageonui carmen button watch remote-control dont boyle tail woods lawn weak-stomached elves rationalism
1 story short mysterious laurier metamorphosis scene strange sewer chainsaw person cinematically benji rowlands progresses resolution starkly twitch dimension roth unearthly
2 1408 uncertainty recovery dont talked funeral staples idea outcasts plot repopulate shouts department hour gozu e-bay unease abuses slicks declining
3 character role gleeful story convict sarsgaard instance plot-heavy petty live burned biblical room catwalk inexplicable wood mentally wee hoodoo meandering
4 hard-pressed watch feel minami ism yoshimi frightened tab se7en essence slapstick two-story police rarely walls audience vampire centre days gate
5 play fun place rides exit feel time angina jolts reveal confessions produce someones trash calmly george ein suspects slam-bang ignite
6 time grotesque favorite think attacks diner mix visitors girls paste fault scandal ride post-birthday titanic tommy wonder issue disturbance starting
7 feel pretty styx icon abandonment tight novelized faces 911 people korea archetype mind-blowing higher fears horrifying horror-thrillers simple-minded et malice
8 irreversible theme wonderous le imbalanced adage strong london-town hollow su-mi resist jungle expense stringy story greatness jeong-min eager red-eye zack
9 work time look mr truth chainsaws isi jesses paula jurassic institute shaman 2010 couple fukikoshi shower estrange toro fanatical lost--this
10 new resistance chance texas begun misunderstand sleep eery safe advice story health deep jeonmal carolyn kiddie porn archetype shouting stinker
11 mist handel based peasant foggiest silk misshapen response require murder thailand acceptable plot critic lawyer twitchers freshment choose sanitize heisman
12 similar trade deteriorate marriage cops neighbours human decoration nullify check-out bright parallel electric trier scene exaggerate leaver plural hellraiser rich
13 cusack purikitpanya character push bolt ing faith negotiable stillness hurricane think udo payback greatness frantic limp professor puzzle keane real
14 edition behaviors adage melodramatics impeccably secretary cock produce iranian nights finally lineage constantly peace theatrical unsettling spar breathtaking heighten taper
15 ripped-to-shreds victorian clich tightly-wired limbless outer stylization meshes suppress whilst service refuge attempt unnatural hillside disturb provide off-guard atmosphere ls054808375
16 perversely home time bewilder people dark characters so-so vampire states wolf loser muse actual people terrifying obnoxious help spring increase
17 noise ameliorate seek titular chase afterthought dont grizzly think boast festival till relaxed connections under-30 consecutive intend gunshot dreamy perplex
18 progress look story similarities money-obsessed self-sacrifice lippi hills diverse grainy somethings unexpectedly decent quarantine oversights foul-mouthed depraved plan jumped rapidly
19 bubblegum dubbed support deceitfully atv reveal phobia spring epilepticmoondancer genitals dispatch palmas 1408 allow third-wheel basking wes 80s love exist
In [117]:
hdp_hellinger_topic_distances = get_most_similar_topics(hdp, hdp_df, hdp_flag=True)
pretty_print(hdp_hellinger_topic_distances)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.18165902124584954, 'Topic # 06', 'Topic # 19')
 (0.18165902124584954, 'Topic # 01', 'Topic # 19')
 (0.1802775637731995, 'Topic # 17', 'Topic # 19')
 (0.1802775637731995, 'Topic # 06', 'Topic # 17')
 (0.1802775637731995, 'Topic # 01', 'Topic # 17')
 (0.17748239349298853, 'Topic # 19', 'Topic # 20')
 (0.17748239349298853, 'Topic # 06', 'Topic # 20')
 (0.17748239349298853, 'Topic # 02', 'Topic # 19')
 (0.17748239349298853, 'Topic # 02', 'Topic # 06')
 (0.17606816861659014, 'Topic # 18', 'Topic # 19')
 (0.17606816861659014, 'Topic # 17', 'Topic # 20')
 (0.17606816861659014, 'Topic # 06', 'Topic # 18')
 (0.17606816861659014, 'Topic # 02', 'Topic # 17')
 (0.17606816861659014, 'Topic # 01', 'Topic # 18')
 (0.17606816861659014, 'Topic # 01', 'Topic # 06')
 (0.17464249196572987, 'Topic # 17', 'Topic # 18')
 (0.17464249196572987, 'Topic # 16', 'Topic # 19')
 (0.17464249196572987, 'Topic # 14', 'Topic # 19')
 (0.17464249196572987, 'Topic # 13', 'Topic # 19')
 (0.17464249196572987, 'Topic # 06', 'Topic # 16')
 (0.17464249196572987, 'Topic # 06', 'Topic # 14')
 (0.17464249196572987, 'Topic # 01', 'Topic # 20')
 (0.17320508075688779, 'Topic # 16', 'Topic # 17')
 (0.17320508075688779, 'Topic # 14', 'Topic # 17')
 (0.17320508075688779, 'Topic # 13', 'Topic # 17')
 (0.17320508075688779, 'Topic # 12', 'Topic # 19')
 (0.17320508075688779, 'Topic # 11', 'Topic # 19')
 (0.17320508075688779, 'Topic # 10', 'Topic # 19')
 (0.17320508075688779, 'Topic # 06', 'Topic # 12')
 (0.17320508075688779, 'Topic # 06', 'Topic # 11')
 (0.17320508075688779, 'Topic # 02', 'Topic # 20')
 (0.17320508075688779, 'Topic # 01', 'Topic # 12')
 (0.17175564037317673, 'Topic # 18', 'Topic # 20')
 (0.17175564037317673, 'Topic # 12', 'Topic # 17')
 (0.17175564037317673, 'Topic # 11', 'Topic # 17')
 (0.17175564037317673, 'Topic # 10', 'Topic # 17')
 (0.17175564037317673, 'Topic # 06', 'Topic # 13')
 (0.17175564037317673, 'Topic # 06', 'Topic # 08')
 (0.17175564037317673, 'Topic # 02', 'Topic # 18')
 (0.17175564037317673, 'Topic # 01', 'Topic # 16')
 (0.17175564037317673, 'Topic # 01', 'Topic # 14')
 (0.17175564037317673, 'Topic # 01', 'Topic # 08')
 (0.17175564037317673, 'Topic # 01', 'Topic # 02')
 (0.17029386365926405, 'Topic # 16', 'Topic # 20')
 (0.17029386365926405, 'Topic # 15', 'Topic # 19')
 (0.17029386365926405, 'Topic # 14', 'Topic # 20')
 (0.17029386365926405, 'Topic # 13', 'Topic # 20')
 (0.17029386365926405, 'Topic # 08', 'Topic # 17')
 (0.17029386365926405, 'Topic # 06', 'Topic # 10')
 (0.17029386365926405, 'Topic # 02', 'Topic # 13')
 (0.16881943016134138, 'Topic # 16', 'Topic # 18')
 (0.16881943016134138, 'Topic # 15', 'Topic # 17')
 (0.16881943016134138, 'Topic # 14', 'Topic # 18')
 (0.16881943016134138, 'Topic # 13', 'Topic # 18')
 (0.16881943016134138, 'Topic # 12', 'Topic # 20')
 (0.16881943016134138, 'Topic # 11', 'Topic # 20')
 (0.16881943016134138, 'Topic # 10', 'Topic # 20')
 (0.16881943016134138, 'Topic # 08', 'Topic # 19')
 (0.16881943016134138, 'Topic # 02', 'Topic # 12')
 (0.16881943016134138, 'Topic # 02', 'Topic # 10')
 (0.16733200530681516, 'Topic # 13', 'Topic # 14')
 (0.16733200530681516, 'Topic # 11', 'Topic # 18')
 (0.16733200530681516, 'Topic # 10', 'Topic # 18')
 (0.16733200530681516, 'Topic # 08', 'Topic # 20')
 (0.16733200530681516, 'Topic # 06', 'Topic # 15')
 (0.16733200530681516, 'Topic # 02', 'Topic # 16')
 (0.16733200530681516, 'Topic # 02', 'Topic # 08')
 (0.16733200530681516, 'Topic # 01', 'Topic # 15')
 (0.16733200530681516, 'Topic # 01', 'Topic # 11')
 (0.16733200530681516, 'Topic # 01', 'Topic # 10')
 (0.16583123951777004, 'Topic # 15', 'Topic # 20')
 (0.16583123951777004, 'Topic # 12', 'Topic # 16')
 (0.16583123951777004, 'Topic # 12', 'Topic # 14')
 (0.16583123951777004, 'Topic # 12', 'Topic # 13')
 (0.16583123951777004, 'Topic # 11', 'Topic # 13')
 (0.16583123951777004, 'Topic # 10', 'Topic # 16')
 (0.16583123951777004, 'Topic # 10', 'Topic # 14')
 (0.16583123951777004, 'Topic # 08', 'Topic # 18')
 (0.16583123951777004, 'Topic # 03', 'Topic # 19')
 (0.16583123951777004, 'Topic # 01', 'Topic # 13')
 (0.16431676725154987, 'Topic # 15', 'Topic # 18')
 (0.16431676725154987, 'Topic # 13', 'Topic # 16')
 (0.16431676725154987, 'Topic # 12', 'Topic # 18')
 (0.16431676725154987, 'Topic # 11', 'Topic # 12')
 (0.16431676725154987, 'Topic # 10', 'Topic # 12')
 (0.16431676725154987, 'Topic # 10', 'Topic # 11')
 (0.16431676725154987, 'Topic # 09', 'Topic # 19')
 (0.16431676725154987, 'Topic # 08', 'Topic # 16')
 (0.16431676725154987, 'Topic # 08', 'Topic # 14')
 (0.16431676725154987, 'Topic # 08', 'Topic # 13')
 (0.16431676725154987, 'Topic # 06', 'Topic # 09')
 (0.16431676725154987, 'Topic # 03', 'Topic # 17')
 (0.16431676725154987, 'Topic # 02', 'Topic # 14')
 (0.1627882059609971, 'Topic # 13', 'Topic # 15')
 (0.1627882059609971, 'Topic # 11', 'Topic # 16')
 (0.1627882059609971, 'Topic # 11', 'Topic # 14')
 (0.1627882059609971, 'Topic # 08', 'Topic # 11')
 (0.1627882059609971, 'Topic # 08', 'Topic # 10')
 (0.1627882059609971, 'Topic # 04', 'Topic # 19')
 (0.1627882059609971, 'Topic # 04', 'Topic # 06')
 (0.1627882059609971, 'Topic # 02', 'Topic # 15')
 (0.1627882059609971, 'Topic # 02', 'Topic # 11')
 (0.16124515496597103, 'Topic # 14', 'Topic # 16')
 (0.16124515496597103, 'Topic # 07', 'Topic # 19')
 (0.16124515496597103, 'Topic # 06', 'Topic # 07')
 (0.16124515496597103, 'Topic # 04', 'Topic # 17')
 (0.16124515496597103, 'Topic # 03', 'Topic # 20')
 (0.16124515496597103, 'Topic # 01', 'Topic # 09')
 (0.15968719422671315, 'Topic # 15', 'Topic # 16')
 (0.15968719422671315, 'Topic # 14', 'Topic # 15')
 (0.15968719422671315, 'Topic # 10', 'Topic # 13')
 (0.15968719422671315, 'Topic # 09', 'Topic # 20')
 (0.15968719422671315, 'Topic # 09', 'Topic # 17')
 (0.15968719422671315, 'Topic # 08', 'Topic # 15')
 (0.15968719422671315, 'Topic # 08', 'Topic # 12')
 (0.15968719422671315, 'Topic # 07', 'Topic # 17')
 (0.15968719422671315, 'Topic # 03', 'Topic # 18')
 (0.15968719422671315, 'Topic # 03', 'Topic # 06')
 (0.15968719422671315, 'Topic # 02', 'Topic # 09')
 (0.158113883008419, 'Topic # 12', 'Topic # 15')
 (0.158113883008419, 'Topic # 11', 'Topic # 15')
 (0.158113883008419, 'Topic # 10', 'Topic # 15')
 (0.158113883008419, 'Topic # 09', 'Topic # 18')
 (0.158113883008419, 'Topic # 04', 'Topic # 20')
 (0.1565247584249853, 'Topic # 09', 'Topic # 16')
 (0.1565247584249853, 'Topic # 09', 'Topic # 14')
 (0.1565247584249853, 'Topic # 09', 'Topic # 13')
 (0.1565247584249853, 'Topic # 05', 'Topic # 19')
 (0.1565247584249853, 'Topic # 05', 'Topic # 06')
 (0.1565247584249853, 'Topic # 04', 'Topic # 18')
 (0.1565247584249853, 'Topic # 03', 'Topic # 12')
 (0.1565247584249853, 'Topic # 01', 'Topic # 05')
 (0.1565247584249853, 'Topic # 01', 'Topic # 04')
 (0.1549193338482967, 'Topic # 09', 'Topic # 12')
 (0.1549193338482967, 'Topic # 09', 'Topic # 10')
 (0.1549193338482967, 'Topic # 07', 'Topic # 18')
 (0.1549193338482967, 'Topic # 05', 'Topic # 17')
 (0.1549193338482967, 'Topic # 03', 'Topic # 16')
 (0.1549193338482967, 'Topic # 03', 'Topic # 14')
 (0.1549193338482967, 'Topic # 03', 'Topic # 13')
 (0.1549193338482967, 'Topic # 03', 'Topic # 08')
 (0.1549193338482967, 'Topic # 02', 'Topic # 04')
 (0.15329709716755896, 'Topic # 08', 'Topic # 09')
 (0.15329709716755896, 'Topic # 07', 'Topic # 20')
 (0.15329709716755896, 'Topic # 07', 'Topic # 16')
 (0.15329709716755896, 'Topic # 07', 'Topic # 14')
 (0.15329709716755896, 'Topic # 04', 'Topic # 12')
 (0.15329709716755896, 'Topic # 04', 'Topic # 10')
 (0.15329709716755896, 'Topic # 03', 'Topic # 11')
 (0.15329709716755896, 'Topic # 03', 'Topic # 10')
 (0.15329709716755896, 'Topic # 02', 'Topic # 07')
 (0.15329709716755896, 'Topic # 01', 'Topic # 03')
 (0.15165750888103105, 'Topic # 09', 'Topic # 15')
 (0.15165750888103105, 'Topic # 09', 'Topic # 11')
 (0.15165750888103105, 'Topic # 07', 'Topic # 12')
 (0.15165750888103105, 'Topic # 07', 'Topic # 11')
 (0.15165750888103105, 'Topic # 05', 'Topic # 20')
 (0.15165750888103105, 'Topic # 04', 'Topic # 16')
 (0.15165750888103105, 'Topic # 04', 'Topic # 14')
 (0.15165750888103105, 'Topic # 04', 'Topic # 13')
 (0.15165750888103105, 'Topic # 04', 'Topic # 08')
 (0.15165750888103105, 'Topic # 02', 'Topic # 05')
 (0.15165750888103105, 'Topic # 02', 'Topic # 03')
 (0.15165750888103105, 'Topic # 01', 'Topic # 07')
 (0.15000000000000002, 'Topic # 07', 'Topic # 08')
 (0.15000000000000002, 'Topic # 05', 'Topic # 18')
 (0.15000000000000002, 'Topic # 03', 'Topic # 15')
 (0.14832396974191328, 'Topic # 07', 'Topic # 15')
 (0.14832396974191328, 'Topic # 07', 'Topic # 10')
 (0.14832396974191328, 'Topic # 05', 'Topic # 16')
 (0.14832396974191328, 'Topic # 05', 'Topic # 14')
 (0.14832396974191328, 'Topic # 05', 'Topic # 13')
 (0.14662878298615184, 'Topic # 07', 'Topic # 13')
 (0.14662878298615184, 'Topic # 05', 'Topic # 12')
 (0.14662878298615184, 'Topic # 05', 'Topic # 11')
 (0.14662878298615184, 'Topic # 05', 'Topic # 10')
 (0.14662878298615184, 'Topic # 04', 'Topic # 15')
 (0.14662878298615184, 'Topic # 04', 'Topic # 11')
 (0.14662878298615184, 'Topic # 03', 'Topic # 09')
 (0.1449137674618944, 'Topic # 05', 'Topic # 08')
 (0.14317821063276356, 'Topic # 05', 'Topic # 15')
 (0.14142135623730953, 'Topic # 07', 'Topic # 09')
 (0.14142135623730953, 'Topic # 03', 'Topic # 04')
 (0.13964240043768944, 'Topic # 04', 'Topic # 09')
 (0.13784048752090225, 'Topic # 03', 'Topic # 05')
 (0.13601470508735444, 'Topic # 05', 'Topic # 09')
 (0.13601470508735444, 'Topic # 04', 'Topic # 07')
 (0.13601470508735444, 'Topic # 03', 'Topic # 07')
 (0.1341640786499874, 'Topic # 04', 'Topic # 05')
 (0.13228756555322954, 'Topic # 05', 'Topic # 07')

______________________________________________________________________
In [118]:
hdp_dom_topics_df, hdp_dom_topics_final = get_dominant_topics(hdp, corpus, all_reviews_joined)
In [119]:
#hdp_dom_topics_final.style.applymap(color_green).applymap(make_bold)
hdp_dom_topics_final_first30 = hdp_dom_topics_final.head(30)
hdp_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Out[119]:
Dominant_Topic Probability_Contribution Topic_Keywords Original_Text
Document_Number
0 3 0.9912 prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times
1 48 0.6778 bland, mani, onijala, multi-faceted, crawl, meaning-symbolism, keyboard, favourite, metaphor, sarsgaard, 1765, honk, elses, iowa, guy-friend, gladiator, treatment, 140, circumstances, notice spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let
2 28 0.9969 story, kurt, query, misspelling, recipient, work, rheda, sealed, steve, evict, act, flying, drink, daughter, nuts, gimmicks, scarcity, titillating, overlap, cartoony wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute
3 5 0.9157 water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole
4 0 0.5896 room, story, 1408, character, hard-pressed, play, time, feel, irreversible, work, new, mist, similar, cusack, edition, ripped-to-shreds, perversely, noise, progress, bubblegum want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems
5 96 0.8354 head-on, carrier, formulaic, roughly, rendered, similar, celebrations, check-out, methodically, rosa, diva, lastly, veteran, narratively, chude, nagoya, asinine, ananda, critically, mimicking ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue
6 10 0.9913 watch, cinematically, repopulate, burned, slapstick, confessions, fault, korea, resist, institute, story, thailand, bright, think, finally, service, wolf, festival, somethings, dispatch virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome
7 3 0.984 prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended
8 17 0.9922 weak-stomached, dimension, abuses, wee, centre, suspects, issue, simple-minded, eager, toro, archetype, choose, plural, puzzle, breathtaking, off-guard, help, gunshot, plan, 80s 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong
9 5 0.9968 water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate
10 11 0.9355 remote-control, benji, shouts, biblical, two-story, produce, scandal, archetype, jungle, shaman, health, acceptable, parallel, udo, lineage, refuge, loser, till, unexpectedly, palmas let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+
11 6 0.8893 character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder
12 12 0.9088 dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty
13 57 0.896 compilation, dvds, unquestionably, nanchuka, penikett, incredibly, restriction, choppy, whisky, lot-, bonkers, ducournau, marvellous, shift, icy, represents, control, hauser, entertains, shortcoming people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original
14 72 0.9758 lowlands, chests, rapid, condoms, christos, airbender, doubt, watched, sheet, 13-year, portion, onethe, dimension, bleeding, brents, antecedent, la, sunlight, well-to-do, autobiographical fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch
15 1 0.5361 story, short, uncertainty, role, watch, fun, grotesque, pretty, theme, time, resistance, handel, trade, purikitpanya, behaviors, victorian, home, ameliorate, look, dubbed zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played
16 103 0.9617 diverse, answer, nod, outdo, roadwork, depths, hath, pm, surrealism, pruitt, ambiguous, laboratory, outside, vampired, videos, slow-buring, stormy, infernal, filmophile, early different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat
17 42 0.9646 light, decisions, monikered, harvey, chockful, excitement, sleeping, manners, behavioral, acted, large, launcher, occupant, shockers, vagina, stared, efficiency, spoilerish, westerner, maximum train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader
18 136 0.9722 actors, humble, compress, chat, close-ups, villains, 3½, thrilling, let, oft, devil, staple, conscience, expected, digitally, relevant, xanthe, takami, romance, minimal dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak
19 4 0.9183 park, metamorphosis, talked, convict, ism, exit, attacks, abandonment, imbalanced, truth, begun, foggiest, cops, bolt, impeccably, limbless, people, chase, money-obsessed, atv zombie plenty thistle cheerleader splendid gods gift debut story peter end
20 72 0.8413 lowlands, chests, rapid, condoms, christos, airbender, doubt, watched, sheet, 13-year, portion, onethe, dimension, bleeding, brents, antecedent, la, sunlight, well-to-do, autobiographical ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring
21 64 0.7754 vanya, reverse, cross, slowly, pig-boy, revenge, gory, pumpkin, rosary, mouth, olin, alaska, crape, opt, likeable, cody, stroke, drum, blasting, persistently dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint
22 47 0.7383 morally-reprehensible, invitation, ism, heartless, martinet, meek, admission, transfusion, cheol-so, outing, look-alike, cross, enthralled, mushroom, finest, elves, gratitudes, silk, assistant, sullen ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced
23 2 0.9494 scare, mysterious, recovery, gleeful, feel, place, favorite, styx, wonderous, look, chance, based, deteriorate, character, adage, clich, time, seek, story, support key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house
24 12 0.9914 dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time
25 21 0.991 dry-ed, rashidi, immobile, salinsageonui, fun-loving, connect, necessary, cold-hearted, saw, operating, lifestyle, faceless, potholing, official, click, happens, high, copious, abstract, cases pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction
26 6 0.9917 character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer
27 95 0.702 stance, bowl, mike, uncover, deadi, ted, marsha, draining, meager, holden, dalle, justify, derrickson, thats, easy-to-spot, kuroki, bliss, character, weakness, 70 admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black
28 91 0.7345 farmhouses, cursory, reopen, hot, 1986, zombie, colourings, reviews, distill, finesse, purchased, fearful, bombarding, diverge, bin, booth, loose, engage, grasped, position doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point
29 146 0.973 cody, flinch, act, resurrect, intensify, drench, solidly, depressing, tatiana, soundproof, linger, nosferatu, opening, happen, persist, se7en, special, explosion, favors, horror- open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
In [120]:
hdp_most_representative_docs = get_most_representative_docs(hdp_dom_topics_final, n_topics=20)
hdp_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[120]:
Probability_Contribution Topic_Keywords Most_Representative_Document
Topic_Number
0 0.3079 room, story, 1408, character, hard-pressed, play, time, feel, irreversible, work, new, mist, similar, cusack, edition, ripped-to-shreds, perversely, noise, progress, bubblegum anthology sum parts particular set stack novice young man trouble past leave live monks karma catch interesting idea lack execution bite bore end ward injure teenager spend night hospital room brain dead old man respirator slate life support day spooky violent twist ending backpackers favorite bunch young hitchhiker pick trucker cargo dangerous id love salvage dishonest car saleswoman terrify night vehicle return angry customer son disappears gruesome ghost story feature karma main theme end funny subversive tale spooky long-haired ghost girl east frequently 10+ years phobia rocky start overall entertain vary mix thai
1 0.2765 story, short, uncertainty, role, watch, fun, grotesque, pretty, theme, time, resistance, handel, trade, purikitpanya, behaviors, victorian, home, ameliorate, look, dubbed poke fun general youre look cradle scare lady-friend watch southpark britney spears episode parallel matt stone trey parker draw modern culture glorify celebrity ancient sacrifice god ensure harvest same- create reality tv type scenario main character walk step clich joker jock slut nerd virgin elaborate version kind sacrifice genre supply endless stream blood satisfy ancient need character clich notice falling cast role human tip hat writers joss clever beasts pure horror- thats point enjoy commentary narrative develope didnt head happening edge interest end point sit think wow fun definitely fun watch
2 0.3555 scare, mysterious, recovery, gleeful, feel, place, favorite, styx, wonderous, look, chance, based, deteriorate, character, adage, clich, time, seek, story, support plot solid entertain mean want new watch genre- entertain lore improve story flashback creative storytelling gem minute doesnt provide time viewer understand evil descend family tell bit piece owner run funeral parlour home owner dagmar hide sell body house build ancient evil leave guess heck rest story old boiler downstairs minute screen time flesh sordid past leave satisfy true understand house evil past leave gore fest jump scare new old fashion star rating
3 0.582 prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully point mince words brad anderson session ive time intelligent well-written completely unpredictable look didnt notice view edit photography work soundtrack downright creepy recently manage lie awake night dario argentous opera tobe hoopers texas chain saw massacre list include honestly excuse folks doesnt
4 0.2166 park, metamorphosis, talked, convict, ism, exit, attacks, abandonment, imbalanced, truth, begun, foggiest, cops, bolt, impeccably, limbless, people, chase, money-obsessed, atv massive fan leave completely cold mean spirited cold unbelievably frustrate stupid times plus act involved lack logic place complete lack satisfy pay merit quickly outweighed example lead character think okay wonder people home consent lead bizarre random mobile phone suspension disbelief lose appropriate response doh ending terrorisation expect decent pay instead downbeats nihilistic sign popular modern miss point somewhat experience unpleasant hate work people
5 0.9157 water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole
6 0.3766 character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia admit excellent scare gory stories novice ward backpackers salvage end novices gory ward scary backpackers thrilling salvage scary gory end funny scary phobia bias bias pawn purikitpanya story bite weird improvement phobia phobia end cast 4bia- middle think funny marsha wattanapanich play funny act funny overreacted thats ism phobia vote
7 0.8265 salinsageonui, sewer, idea, plot-heavy, tab, angina, visitors, faces, london-town, jesses, eery, response, decoration, negotiable, produce, meshes, so-so, grizzly, hills, spring die darks festival japanese chill complex spooky follow nagisa japanese actress base real murder spree place large hotel forty-some year ago excite look forward filming scary begin spooky angina begin ghostly girl vision intensify director cast real hotel actual murder happened inspiration connection hotel grisly murders answer soon havenet asian original ju-on ive remake havenet pretty wasnt expect luckily pleasantly surprised reincarnation grinned original language solid supernatural rise far expectations write nicely inconsistency major midway think figure positive plot twist turn pretty confident figure final minute entire assumption disintegrated leave minor shocked ism rest plot twist coming suppose expect twist set-up trick audience admit work intelligent writing director takashi shimuzu direct ju-on ju-on american remake grudge grudge eye eerie unlike style grudge reincarnation boast subtle spookiness personally menace annoy jump scares numerous shot scene hard forget direction lots creepy shot disturb imagery homemade snuff footage eerie creepy music fit perfectly slow subtle acting-wise id succeed performer pretty believable language barrier think conclusion real kickers surprise creeped entire close overall reincarnation supernatural traditional ghost story throw hotel reincarnation theme youve enjoy slow andor fan japanese id recommend minor problems easily outweigh bad end surprising ism fit tastes couldnt pick foreign flick
8 0.9828 carmen, chainsaw, outcasts, petty, se7en, jolts, girls, 911, hollow, paula, safe, require, nullify, stillness, iranian, suppress, vampire, think, diverse, epilepticmoondancer think creepy tone vibe atmosphere begin minutes think creepy feel clever element throw wrong bore doesnt artistic original approach classic direction falters mainly slow build isnt entice new build pointless drivel lengthen runtime audience member classic movies pace bad overall forgettable flick isnt worth time semi-retro cinematography look potential maybe delve mystery element revolve town people
9 0.4927 button, person, plot, live, essence, reveal, paste, people, su-mi, jurassic, advice, murder, check-out, hurricane, nights, whilst, states, boast, grainy, genitals jeepers creepers common 1950 ec comic book past years fact isnt bad plot idea decent expression decent surprise pace stay interesting identity monster stay mysterious youre totally certain suppose plot intelligently grade dialog isnt stupid contrived rend worth cash
10 0.3416 watch, cinematically, repopulate, burned, slapstick, confessions, fault, korea, resist, institute, story, thailand, bright, think, finally, service, wolf, festival, somethings, dispatch deserve allot praise play norwegian cultural memes visually landscape place folklore troll arose course spice lovely computer graphic needed graphic merge rockumentary style fall categories mix comedy action fake amateur documentary style bad lack bite flow times course drawback apply foreigners norwegian story portuguese girlfriend think okay brief fairy tale joke sheep bridge hard catch appreciated marinate culture time issue cameo appearance norwegian comedians slightly spoil illusion disagree suspect seven norwegians concepts rich despite flaws ll love visual drama norwegians exotic neat offer relief classical hollywood monster depictions curious google theodor kittelsen asbjornsen moed enjoy
11 0.7247 remote-control, benji, shouts, biblical, two-story, produce, scandal, archetype, jungle, shaman, health, acceptable, parallel, udo, lineage, refuge, loser, till, unexpectedly, palmas confessions savage brutal poignant revenge story seen doesnt start end awe begin japanese classroom final day class spring break remainder event follow fateful day classroom point view switch numerous time different people affect event beginning progress revisit past scene different characters perspective scene monotonous time onion confessions multi-faceted discover re-discover subsequent layer peel backra beautiful wildly sympathy flip-flopped dont adamantly learn character event sympathy bind jump ship point youre totally confessions challenge frustrate add confessions charm wit unrelentingly dark bind incite feeling bleakness suit weak heart subject matt heavy character morally-reprehensible feel wholly somber grey dull blue saturate scene melancholy albeit perfectly-suited soundtrack work infectiously mind horrific scene eye resonate deeply spiel longer closing act top-notch storytelling captivating cinematography gorgeous touch violence blood taste immense effect feel gratuitous japanese cake seen ive far sound convince definitely shoot hard let ounce
12 0.3394 dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408 incredibly cruel unrelenting play single feature divide sections dumplings direct fruit chan hong kong cut direct park chan-wook korea box direct mike takashi japan section dissertation dumplings classify exploitation beautifully produce directed box lavish sets costume atmosphere culture respective country provide new terrify appalling original right cut compare saw hannibal letter franchise theres psycho whoas message murder mayhem dont want detail plot think viewer need experience preconception dumplings plot high squeam factor cut cover familiar premise fun moment ll box artistic endeavour plot reason enthrall couldnt look mostra solid effort genius asian directors
13 0.331 boyle, progresses, hour, catwalk, rarely, trash, post-birthday, higher, stringy, couple, jeonmal, critic, trier, greatness, peace, unnatural, actual, connections, quarantine, allow caught resolution festival describe doesnt fit sub-genre definitely belong family creative unique ive thoroughly entertaining pace guess entire time conclusion personally canst wait view lead performance essential isolate setting end people love hated love speak fellow audience member dislike dont end loving ll doubt entertained plus provide interest conversation fellow fans think viewers refresh unique quality hard hard modern wont regret star
14 0.7917 tail, resolution, gozu, inexplicable, walls, calmly, titanic, fears, story, fukikoshi, carolyn, lawyer, scene, frantic, theatrical, hillside, people, under-30, oversights, third-wheel approach adaptation video game extreme trepidation think corker weave catch tv past super mario bros mortal combat resident evil stinkers doom vapid close source materials silent hill open music lift straight game fact entire contain original track game remix score fantastic brood background play game work effectively screens story composite story element games final act monstrous highly original thirty minute guess completely gross violence pretty gruesome clever mixture cgi fully made-up monster add forebode atmosphere feel nervous watch visual absolutely spot ive adaptation close source material comic book past year stray furiously weak opening id resign fact disappointing pleasantly surprise soon main character rose silent hill youre fan game wont disappointed youre hounds ll love creature visuals youre expect deep relationship estrange mother daughter youre wrong theatre highly recommended best ive time
15 0.9711 woods, starkly, e-bay, wood, audience, george, tommy, horrifying, greatness, shower, kiddie, twitchers, exaggerate, limp, unsettling, disturb, terrifying, consecutive, foul-mouthed, basking description think yeah possession yawn waste evening look positive review decide disappointment original material craft cinematic acting plot reasonably scary suspenseful solid recommend strongly fan trust dont recommendation easily plenty crape gold job
16 0.5523 lawn, twitch, unease, mentally, vampire, ein, wonder, horror-thrillers, jeong-min, estrange, porn, freshment, leaver, professor, spar, provide, obnoxious, intend, depraved, wes idea boy pleasantly surprised director rec set apartment block spanish gem cast lead luis play superbly fill increase sense unease shock youre witnessing beautifully shoot took apartment block setting end leave think happened far creepiest twist tale ive pleasure watch time highly recommended
17 0.7554 weak-stomached, dimension, abuses, wee, centre, suspects, issue, simple-minded, eager, toro, archetype, choose, plural, puzzle, breathtaking, off-guard, help, gunshot, plan, 80s ju-on grudge easy america wasnt write review hear hype heaven magazine fangoria rue morgue word mouth finally track la watch chance ju-on chapter story haunt house tokyo suburb begin inexperienced social worker house face face story jump past present chapter focus character time circle unwise enter curse house wind dead haunt spread virus terrible murder place house rage surround act violence spawn evil curses enter house immediately infect haunt follow people home drive near madness drag ju-on bear pass resemblance popular predecessor ringu near frightening bad means butchered mother-ghost kayaks sadako-like crawl black hair face unearthly jerkiness blue-white face startle huge stare eye occasional splash blood ghost son toshio sad frightening appear normal boy pale wide-eyed ghost frighten sequence feature murder woman kayako head black hair peek corner shadow corridor fill security camera head-on shoot crawl attic night beam flashlight illuminate sound effect disturb performance convincingly wasnt scare promise thats happen buy hyper expect pretty ghost story instead ju-on decent straightforward ghost story scary moment fans ring scarier ju-on noble effort asian stories remain ambiguous open-ended leave room sequel chance decide curse grudge stars
18 0.8376 elves, roth, slicks, hoodoo, days, slam-bang, disturbance, et, red-eye, fanatical, shouting, sanitize, hellraiser, keane, heighten, atmosphere, spring, dreamy, jumped, love seeing session reaffirm truly including filmmakers comparison dont look shining nod changeling session stand effective brood experience dread -- thats things style tension genuine grandiose shining session rely real fear gratuitous material entertain want creep thats soul purpose pretty young gap models trendy mtv-influenced rap metal soundtracks breasts giggle-inducing decapitation effects want mean watch plenty offer short attention spans want draw place fear think session therapy session nightmares story simple complex time worker remove asbestos massive danvers mental hospital slowly unravel secret tape audio recording patient characters background underdeveloped complain -- dont need guy information slow nice brood pace right dont worry clues leave unresolved thats fun weird clues david lynches work twin peaks mulholland drive weird draw direct connection main story actual danvers hospital amaze set structure character disturb minimalistic soundtrack unsettle downright perfect buy cd figure attract play background type actor fine job -- yes david caruso need guy bite impatient maintain sense calm peter mullane steven gevedon josh lucas brendan sexton iii real regular blue collar guys refresh insecure mature rough-edged cast sense guy real eyes tend slack bite place theyre suppose job weeks quiet dread story draw ll absorb completely course appreciate flashy ll agree damn near perfect idb criticize scene jar peanut butter leave floor welllll think consider state consumer think leave care container disposed man people picky strange things implausible story characters action doesnt wreck appreciate atmosphere story didnt end hard understand mind set didnt busy angry sit let present clearly categorize ive seen definitely years scare theres specific scene josh experience basement cause wave tingle goosebumps body exhilarate scare effectively single scene folks canst trash doesnt easy explanation allow cheap voyeuristic thrills didnt session read summary wasnt offer slam-bang entertainment mind stimulate love absorb mysterious wondrous storytellings session definition flawless piece work id
19 0.5641 rationalism, unearthly, declining, meandering, gate, ignite, starting, malice, zack, lost--this, stinker, heisman, rich, real, taper, ls054808375, increase, perplex, rapidly, exist fan art drama dont genre understand hate true fan basic knowledge cult history genre real joy cabin woods weird precisely genre determined gather idea countless time turn upside-down new ability original totally outwear cliches parody homage time funny-scary visual dialogue one-liners pace twist predictable let bore minutes final revelation time ridiculous awesome screen masterpiece real gem love
In [121]:
hdp_topic_dist = get_topic_distribution(hdp_dom_topics_final, n_topics=20, hdp_flag=True)
hdp_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[121]:
Documents_per_Topic Percent_of_Total_Corpus Topic_Keywords
Topic_Number
0 66 0.066 room, story, 1408, character, hard-pressed, play, time, feel, irreversible, work, new, mist, similar, cusack, edition, ripped-to-shreds, perversely, noise, progress, bubblegum
1 58 0.058 story, short, uncertainty, role, watch, fun, grotesque, pretty, theme, time, resistance, handel, trade, purikitpanya, behaviors, victorian, home, ameliorate, look, dubbed
2 26 0.026 scare, mysterious, recovery, gleeful, feel, place, favorite, styx, wonderous, look, chance, based, deteriorate, character, adage, clich, time, seek, story, support
3 20 0.02 prof, laurier, dont, story, minami, rides, think, icon, le, mr, texas, peasant, marriage, push, melodramatics, tightly-wired, bewilder, titular, similarities, deceitfully
4 18 0.018 park, metamorphosis, talked, convict, ism, exit, attacks, abandonment, imbalanced, truth, begun, foggiest, cops, bolt, impeccably, limbless, people, chase, money-obsessed, atv
5 16 0.016 water, scene, funeral, sarsgaard, yoshimi, feel, diner, tight, adage, chainsaws, misunderstand, silk, neighbours, ing, secretary, outer, dark, afterthought, self-sacrifice, reveal
6 22 0.022 character, strange, staples, instance, frightened, time, mix, novelized, strong, isi, sleep, misshapen, human, faith, cock, stylization, characters, dont, lippi, phobia
7 14 0.014 salinsageonui, sewer, idea, plot-heavy, tab, angina, visitors, faces, london-town, jesses, eery, response, decoration, negotiable, produce, meshes, so-so, grizzly, hills, spring
8 18 0.018 carmen, chainsaw, outcasts, petty, se7en, jolts, girls, 911, hollow, paula, safe, require, nullify, stillness, iranian, suppress, vampire, think, diverse, epilepticmoondancer
9 16 0.016 button, person, plot, live, essence, reveal, paste, people, su-mi, jurassic, advice, murder, check-out, hurricane, nights, whilst, states, boast, grainy, genitals
10 20 0.02 watch, cinematically, repopulate, burned, slapstick, confessions, fault, korea, resist, institute, story, thailand, bright, think, finally, service, wolf, festival, somethings, dispatch
11 8 0.008 remote-control, benji, shouts, biblical, two-story, produce, scandal, archetype, jungle, shaman, health, acceptable, parallel, udo, lineage, refuge, loser, till, unexpectedly, palmas
12 20 0.02 dont, rowlands, department, room, police, someones, ride, mind-blowing, expense, 2010, deep, plot, electric, payback, constantly, attempt, muse, relaxed, decent, 1408
13 16 0.016 boyle, progresses, hour, catwalk, rarely, trash, post-birthday, higher, stringy, couple, jeonmal, critic, trier, greatness, peace, unnatural, actual, connections, quarantine, allow
14 12 0.012 tail, resolution, gozu, inexplicable, walls, calmly, titanic, fears, story, fukikoshi, carolyn, lawyer, scene, frantic, theatrical, hillside, people, under-30, oversights, third-wheel
15 16 0.016 woods, starkly, e-bay, wood, audience, george, tommy, horrifying, greatness, shower, kiddie, twitchers, exaggerate, limp, unsettling, disturb, terrifying, consecutive, foul-mouthed, basking
16 10 0.01 lawn, twitch, unease, mentally, vampire, ein, wonder, horror-thrillers, jeong-min, estrange, porn, freshment, leaver, professor, spar, provide, obnoxious, intend, depraved, wes
17 8 0.008 weak-stomached, dimension, abuses, wee, centre, suspects, issue, simple-minded, eager, toro, archetype, choose, plural, puzzle, breathtaking, off-guard, help, gunshot, plan, 80s
18 10 0.01 elves, roth, slicks, hoodoo, days, slam-bang, disturbance, et, red-eye, fanatical, shouting, sanitize, hellraiser, keane, heighten, atmosphere, spring, dreamy, jumped, love
19 8 0.008 rationalism, unearthly, declining, meandering, gate, ignite, starting, malice, zack, lost--this, stinker, heisman, rich, real, taper, ls054808375, increase, perplex, rapidly, exist
In [122]:
hdp_topics = []
for topic_id, topic in hdp.show_topics(num_topics=20, formatted=False):
    topic = [word for word, _ in topic]
    hdp_topics.append(topic)
In [123]:
hdp_cm_umass = CoherenceModel(topics=hdp_topics, corpus=corpus, dictionary=dictionary, coherence='u_mass')
print('The U_MASS coherence measure of the HDP Model is: {}'.format(hdp_cm_umass.get_coherence()))
The U_MASS coherence measure of the HDP Model is: -17.870792618102282
In [124]:
hdp_cm_cv = CoherenceModel(topics=hdp_topics, texts=all_reviews_text, dictionary=dictionary, coherence='c_v')
print('The C_V coherence measure of the HDP model is: {}'.format(hdp_cm_cv.get_coherence()))
The C_V coherence measure of the HDP model is: 0.6891603871535945
In [80]:
prepared_hdp = pyLDAvis.gensim.prepare(hdp, corpus, dictionary)
display(pyLDAvis.display(prepared_hdp))

LSI (Latent Semantic Indexing)

Cf. http://lsa.colorado.edu/papers/JASIS.lsi.90.pdf

In [81]:
mod_list_lsi_umass, coh_list_lsi_umass = compute_coherence_values(LsiModel, dictionary, all_reviews_text, max_topics=20, 
                                                                     corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                                                     min_topics=2, stride=3, n_top_words=False, measure='u_mass', 
                                                                     lsi_flag=True)

mod_list_lsi_cv, coh_list_lsi_cv = compute_coherence_values(LsiModel, dictionary, all_reviews_text, max_topics=20, 
                                                                 corpus=corpus, topics=False, transformed_vectorizer=False, tfidf_norm=False,
                                                                 min_topics=2, stride=3, n_top_words=False, measure='c_v', 
                                                                 lsi_flag=True)
Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'gensim.models.lsimodel.LsiModel'> using the U_MASS measure


Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'gensim.models.lsimodel.LsiModel'> using the C_V measure


In [82]:
best_num_lsi_topics = show_best_num_topics('LSI', coh_list_lsi_umass, coh_list_lsi_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of LSI topics to use is: {}'.format(best_num_lsi_topics))
The most coherent number of LSI topics using the U_MASS measure is: 2
The most coherent number of LSI topics using the C_V measure is: 5


The most coherent number of LSI topics to use is: 5
In [83]:
lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=best_num_lsi_topics, chunksize=1000)
In [116]:
#lsi_topics_matrix = lsi.show_topics(formatted=True, num_words=20)
In [117]:
#lsi_topics_matrix = np.array(lsi_topics_matrix)

#lsi_topics_matrix.shape
In [118]:
#lsi_topic_words = lsi_topics_matrix[:,1]
#for i in lsi_topic_words:
#    print([reg_ex.sub('', w) for w in i.split() if len(reg_ex.sub('', w)) > 0])
#    print('\n')
In [84]:
lsi_df = get_topics(lsi, best_num_lsi_topics)
#print(lsi_df)
lsi_df
Out[84]:
Topic # 01 Topic # 02 Topic # 03 Topic # 04 Topic # 05
0 story sarah people water story
1 time juno battle ring original
2 character friend story dark room
3 people cave world original work
4 think subconscious royale yoshimi beast
5 feel action school watch vampire
6 watch deep youth scene ghost
7 scene leave high work 1408
8 end stab island feel battle
9 dont conscious kill world watch
10 work level idea girl french
11 plot people juno ikuko school
12 look kill sarah think character
13 want event videogame time scene
14 doesnt dreamer hate beast remake
15 play wreck love child le
16 love husband young nakada creature
17 director story teenagers young frontal
18 start run die form mike
19 genre death problem people genre
In [85]:
lsi_hellinger_topic_distances = get_most_similar_topics(lsi, lsi_df, lsi_flag=True, num_topics=best_num_lsi_topics)
pretty_print(lsi_hellinger_topic_distances)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.5568771924831382, 'Topic # 01', 'Topic # 05')
 (0.5564700053205395, 'Topic # 01', 'Topic # 04')
 (0.5561245367232094, 'Topic # 01', 'Topic # 03')
 (0.5557440151483, 'Topic # 01', 'Topic # 02')
 (0.02155843423777627, 'Topic # 03', 'Topic # 05')
 (0.021381967767978346, 'Topic # 04', 'Topic # 05')
 (0.01904421110593379, 'Topic # 02', 'Topic # 05')
 (0.01822447869120819, 'Topic # 03', 'Topic # 04')
 (0.01618622518404863, 'Topic # 02', 'Topic # 04')
 (0.015231272000547094, 'Topic # 02', 'Topic # 03')

______________________________________________________________________
In [86]:
lsi_dom_topics_df, lsi_dom_topics_final = get_dominant_topics(lsi, corpus, all_reviews_joined, lsi_flag=True)
In [87]:
#lsi_dom_topics_final.style.applymap(color_green).applymap(make_bold)
lsi_dom_topics_final_first30 = lsi_dom_topics_final.head(30)

print('Note: For the LSI model, the (un-normalized) topic-probability with the largest magnitude is shown! \n')
lsi_dom_topics_final_first30.style.applymap(color_green).applymap(make_bold)
Note: For the LSI model, the (un-normalized) topic-probability with the largest magnitude is shown! 

Out[87]:
Dominant_Topic Probability_Contribution Topic_Keywords Original_Text
Document_Number
0 0 4.5363 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre coherent plot tell non-straightforward fashion open interpretations leave require audience fill-in open end possibly filling-in require viewings considerable think sense watch wailing time impression kind watch ponder read watch attempt people interpret explain different conclusion coherent underlie plot matt shift piece puzzle attempt recreate coherent narrative piece fit incompetence underlie story possibly intentionally possibly fundamentally inconsistent optical illusion escher drawing appear describe physical object fact dont physical sense accordingly enjoy boil content technically well-made cinematography acting costumes plot sense tell deceptive lure think plot sense matt sufficient thought leave lingering irritate feel dissatisfaction confusion maybe precisely point tell story purpose instill audience feel confusion face sequence event sense life times
1 0 2.004 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre spoiler review extraordinary hour technically-perfect humanistic fine writer directors business hauteur saw devil cipher close analog suggest david lynches 2001 mulholland drive technically perfect humanistic suspense opus captivate length walk theatre shake head wonder saw film-makers understand secret story tell present story viewer feel share experience protagonist tell story viewer going provide explanation wailing means re-quote writer director numerous interview enigmatically begin wonder nature god -- recommended levels entertainment puzzled clinic engage wont let
2 0 9.9919 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre wailing open quote bible easy forget fact watch certain point clear purpose quote served warning religious person scare wailing work religious parables overcome lengthy run time tonal imbalanced deliver funny truly terrify mysterious strange small village south korean village plague sickness police think wild mushroom blame police officer jong-goo think stranger meet woman information mysterious man slowly begin fall rabbit hole consume life daughter start sign sickness personal figure trust avoid director hong-jin nap struggle tonal balance time pitch perfect time place tone struggle stay consistent primarily funny brief burst depravity violence comedy work laugh numb times horrific imagery sudden effective balance lose act comedy slapstick fit rest character over-the-top act exaggerate different introduced middle section moment unintentional humor example man strike lighten caller id shaman phone shaman fortunately balance act smartly switch act intense terrify moment comedy offset terrify act start unfold audience gain new appreciation rest start reinterpret certain scenes problem scene sense ended feel natural fit story progressed require end sense hour minutes middle portion resolved shaman screen time perform ritual end important zombie scene feel awkward didnt fit rest want zombie grow popularity scene bring numb random character serve purpose rest despite scene previously mention overacting act generally speak fantastic father task solve happen daughter convey terror hatred build intense persona carry priest train advice father equally effective bring concern innocent quality terror ultimately happen young daughter sick shines channeling inner linda blair emphatically deliver horrible dirty line child performance truly terrify watch hatred eye slowly end religious overtone clearer locusts attack individual white black suggest characters true nature scene truly shines slowly unveil real nature certain characters minute change perspective things minute switch determine trust reveal realize detail setup reveal twist lead ultimately putt faith wrong people wailing 2016 directed hong-jin na screenplay hong-jin na starring woo-hee jeong-min hwang so-yeon range won krak run time hour minute
3 0 3.1264 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre hell ride minute stop look time leave relieved left thats engage interest story watch read review smorgasbord genre positive review doesnt feel means yes storytelling fluid write multiple genre aspect didnt feel place works work feel redundant point genres similar witch necessarily term storyline pace severity terror psychological wailing deliver somewhat complex storyline cinematography breathtaking jun kunimura ridiculous oh-so-good actor point ism point mention asian technically guess lol dont think necessary pigeonhole
4 0 3.4346 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre want start review completely jump scares play integral movies rely support story leave displeased wailing special jump scare instead viewers story rich mystery character struggles dark atmosphere design amazingly craft scene blood run cold addition carry subtext leave viewer question evidence interpret feel director time research real experience possible wailing bite bore slow burner construct modern running time th 36min patient man pay end genre dead look cash grab mainstream wide release hide gems
5 0 6.1234 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre ism biased set city live work oxford street piccadilly circus pass morning usually teem crowd people completely send shiver spine usually watch locate nondescript midwestern village easy detach event unfold screen occur place home entirely new sense reality previously unaccustomed judge days entirely merit easy arrive conclusion fantastic achievements coming-of-age sort director danny boyle canst mtv-inspired vanity beach self-consciously trendy posture trainspotting appeal shame initially expect days similar treatment thankfully fear prove unfounded discard straight open sequence effortless fearsome rest joy terrify joy joy nonetheless true minimalism effective overblown bravado definitely true scene complete silence entire metropolis grainy picture serve add documentary-style quality situation real bear definitely wise choice digital video occasionally meet people think days wasnt scary gory people tell 2001 boring memento confusing ignore didnt understand purpose confuse change pace feel distract personally regard important reviewer point succinct point scarier end world world repopulate maniacs think real days lies 28 days romero zombie flick yore ultimately allegory day live age constantly confront violence medium ape right start violence beget violence humanity face uncertain future applaud danny boilers bravery days undoubtedly commercial risk majority cinema-going public prefer escapism word caution remember rage human-made disease allegory masterpiece time days misunderstand considerable people doubt history classic perfectly sum confuse era live didnt advisable days chance haunt experience look right angle danny boyle year leave hope hell continue
6 0 1.9574 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre virus rage virus infect person mad extreme rage hungry blood day outbreak london cause entire britain dead evacuate leave blood-thirsty infect population handful solitary normal persons civilization halt society destroy limit survivor fight existence frequent attack vicious victims sounds familiar 28 days classic horde zombie biohazard movies touch art danny boyle able bring focus extensive special-effects-controlled gory action sequence infect normals heavy background music theres tinge sadness emptiness helplessness consider london scene background music theres electrify action gore describe picture life condition talk mistake loophole couldnt weaken tight-gripping storyline achievement viewer stay neutral place virus talk metaphor rage social disease flick person harsh critic zombie watch stars mention billian murphy awesome
7 0 1.7303 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre ism amaze negative review think succeed level artistic atmospheric pace sympathetic characters fantastic climax music nicely eerie open scene london street worth price admission ism stubborn viewer normally benefit early critical buzz manner excuse time ism completely impressed incidentally think interest day inspire knockoffs knockoffs 28 days apparently owe john wyndham classic disaster novel day triffids thats things highly recommended
8 0 2.5955 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre 2003 state-side release danny boilers 28 days advertise chockful scare-fest didnt day ago gotta feel somewhat embellishment promoters environmental terrorist attack lab contain diseased chimp infect rage virus unwittingly let loose plague lay waste england rest society days title cut abandon london coma-tized bicycle courier jim cillian murphy effective wake stasis hospital search london sign life rescue rage zombie couple survivalists lovely naomi harris follow place place alive meet man daughter brendan gleeson terrific megan burns refuge london-town record message paradise salvation track frank man shortwave radio feel meditation happen people reduce low elements friend tell movies run zombie inspire zombie remake dawn dead dawn pretty full-throttle action hybrid start finish play emphasis creature lucky survivors disturb lesson nature survival interest disturb flick sell wrong
9 0 6.7229 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre key sci-fi genre alive cinemas material technique filmmakers present competent average creative havenet havenet style form george romero prime era film-making bring forth memorable trilogy time genre consider romero list director included important source independent 1968 inventive result masterpiece lacklusters director danny boyle author alex garland cook yarn similar romero wouldnt satisfying theyve essential success days later- idea practice years turn fresh audience feel repelled excited terrified nauseous enthralled wont leave feel weave complete hack work boyle team set freshen zombie string precise term zombie movie- hear living-dead utter hear infected new word people rage infect britain monkey virus let loose island begin infectious spread cut man jim lie hospital bed wander abandon street view tear fragment society boyle implement atmosphere heavily action chase scene indication dedication details jim soon survivors include selena naomie harris father daughter brendan gleeson megan burns hear salvation radio decide brave military outpost thats true salvation outside military typos boyle producer andrew macdonald spellbind thats proper terminology adaptation trainspotting craft par arguably toppings story characters captivate care plight jim companions photography anthony dod mantle striking digital photography blair witch use non-professional camera equipment add proper shade needed shoot composition different edit chris gill quick expect attack scenes fast infect throw blood scream on-looker new infect transform seconds result follow mark tildesley production designs john murphys music evoke haunting evocative mood mundane scenes acting consider actor well-known believable script ism days cup tea science fiction fan immediately hear preview tv add dismiss phooey rubble borrow video-store point view similarity romero military scene remind day dead chain zombie practical reasons supermarket scene unneeded consider satirical reverence dawn dead understand boyle isnt 100 original point genres history credible job different tone different set country different envelop view scene structures overall days construct execute sci-fi youve seen sci-fi youve combined sense modern audience fascinate
10 0 5.2701 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre let right vampire seen smarter scary nuanced doesnt feel thriller feel literature detail bizarre misadventure pair pre-teen star cross lovers androgynous vampired phenomenal regard detail young oskar kare hedebrant life spot stuck incredibly painful period post-childhood pre-adolescence oskar aware girls idea contend small age brutalize boy result terribly collect news clip violent crime let rage day strange young girl eli klina leandersson appear playground fast friend begin look oskar eli innocently spend night occasion privy oskar doesnt happens elias caretaker serial killer brutal order desanguinating victim bucket soon oskar realize new friend bite tragedy shock violence eli leave fend desperately stave urge drink fresh blood form delicate new bond oskar remake let right dont wait hope capture magic original gorgeously shot impeccably written fully realize unmatched respect vampire lore acting producer amaze young actors odd recapture brilliant melancholy chemistry astronomical let right think traditional opt endless ultra violence cut favor kiddie friendly rating director tomas alfredson steer line right middle violence brutal horrific dwell leave question kidney display stillness mimic oskar stage stuck moving hope growth begin changes apparent stillness oskar elik oskar grow change man eli stick burdensome fate quiet understate ending haunting whimsical forth times people understanding kind people beg dont miss time silence lambs real chance home oscar gold deserve a+
11 0 4.4798 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre let right heart sweet coming-of-age story unique different defy categorizations swedish adapt john aside lindqvist bestselling book director tomas alfredson dare mix pleasure pain horrify tender let right storyline reveal secret early leave surprise necessarily rare review plot better 12-year-olds oskar kare hedebrant eli klina leandersson meet snowy afternoon jungle gym courtyard oskar house complex outside stockholm young tender attraction apparent right start think relationship headed deep dark secret discover reveal audience repulse curiously fascinate time similar fashion yellow crime scene tape bring close warn support cast completely beholden narrative revolve adorable young couple performance rival ive actor age innocence vulnerability hedebrant oskar tour-de-force admirably carry shoulders leandersson match scene scene line line result literally chills production value stellary technical aspect -- lighting original music johan soderqvist hoyle van hoytema cinematography -- combine perfect synchronization produce hitchcockian tale bring love light dark drama imaginable let right overwhelm choice best narrative feature north american premiere 2008 tribes festival truly well-deserved honor tomas alfredson craft brilliant work art leave shake head wonder
12 0 3.4355 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre tomas alfredson let right original dark twist gory fantasy special hard classify merely exercise style brilliant piece amoral storytellings characters action defy logic common sense dont wanna spoil moment ll mean revenge moment story happens remind fantasy tale oskar kåre hedebrant year-old bully boy befriend develop innocent crush new neighbor eli klina leandersson happen vampired twist tale revenge pubescent love visual flair swim pool scene classic creative direct impressive performance young pair protagonists hollywood course didnt waste time announce upcoming remake lazy read subtitles likely remake turn pg-13 order money fill moral value prudish parent let kid watch dont vampired year ago ignore future bomb enjoy original youre treaty
13 0 6.0132 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre people review idb focus sweet friendship year old human vampire leads huge element sweet story childhood friendship godfather story father concern worry kids true descriptions miss point think seen truth let right dark measure brilliant story tell week ago disturb echo stay refuse enjoy prior knowledge impossible review properly disclose plot elements warn spoilers ahead comment said character hokan key story end lead believe oskar step place fact reveal final scene dark nightmarish core blossom friendship witness prior offer redemption oskar path relationship eli takes håkan butcher boy old oskar food elik story storytelling high order dark whisper hint suggest questions allude play mind answered hokan pick victim presumably elias preference elias interest oskar elias true nature time briefly glimpse true physical age fact late middle age roughly age håkan gift oskar moments brutal slaughter child tormentors entranced glamour oskar happy happy happily sunset oskar presumably deaden soul destroy husk person hokan butcher fellow human provide eli food puzzle people think sweet fact conclusion utterly chilling oskar happily swap commonplace misery childhood bully fate truly hell earth happily smile shoot doesnt clue superlative heartily recommends tale succeed suggestion play imagination truly addition vampire cannon variation bram stokers brilliant original
14 0 1.1521 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre fond vampire genre artistic poetic profound explore nature evil world child pure evil pure goodness somewhat discernible achieve astound array contrast allow evil coexist thought-provoking leave audience yearn read novel short gems need remake accord dont need wait 2010 watch
15 0 0.6886 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre zombies zombies cheap outta original idea moves mandatory width train play beautiful girl sucker long-haired woman short skirts babe morality story ethic means played
16 0 0.5323 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre different zombies army undead create industrial mishap result overwhelm chaos danger address morality root cause personally love pretty woman enjoy leg beautiful hair girl hat
17 0 1.1526 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre train susan treat mind eyes look capitalism vs humanity man vs empathy remember time enjoy foreign act excellent effeminate young man story original definitely eye delicious cheerleader
18 0 0.5609 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre dont youths fun action-packed dangerous elements innovative pretty girl importantly successful yup hollywood remake entry series redundant unnecessary inferior whitewash woman western taliban style ugly hollywood remakes paul fig jar jar abrams negotiate cut speak
19 0 0.4347 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre zombie plenty thistle cheerleader splendid gods gift debut story peter end
20 0 8.7146 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre ve guess director saw end inventive filmmaker work industry james wan brilliantly retro day deliver extremely stylistic visually strike stand tall classics virtually sex gore curse conjuring earn r-rating scare set 1971 conjuring focus marry paranormal researcher ed patrick wilson lorraine vera farmiga warren lecture college interest case theyre think retirement cue perron family parent roger iron livingstone carolyn lili taylor scare live live daughter claim evil rhode island home doesnt warrens discover perrons torment supernatural wanting short conjuring terrify ive seen trying erase torture porn crowd prove difficult director saw doubt finally look pg normally frown junkiest despite sex gore swearing james wants late slap r rate youre wonder frighten think mpa speak behalf day climax middle turn follow doesnt affect conjuring comic relief brilliantly place bring climax chance build fear startle wan understand psychology tension build suspense mere scene construction obviously note exorcist amityville inspiration derive real case file warren famous case date straying ironic style famous cabin woods surface story inventive assure work creative recent memory ive love james wan manage unoriginal haunted-house- possession story cases weave scares pacing sound design camera work describe precise james wan cinematographer john leonetti responsible look insidious fresh visual style unlike include wide shot movement cinematography absolutely stunning rely shaky cam realism leave unique feel separate visually narratively relate toone issue ive recent genre reveal demon insidious sinister example hook story fear unknown filmmaker audience wan definitely learn insidious conjuring apparition arendt reveal audience theyre far focus letting use imagination truly horrifying think filmmakers note mrs wan want complaint isnt wrong market screw trailer conjuring reveal scares avoid trailer command james wants twitter account hard miss tv spots wish fresh mind havenet trailer want stay marketing ultimately overall production value allow conjuring stand rot genre act impressive practical effect perfect classic feel wan time movies must-see summer rating 10let twitter thejoshl think conjuring
21 0 4.0121 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre dont summon devil dont priest lucky conjuring preview screen brightest 2013 totally cold trailer story plot turn scary seen conjuring nail-biting hiding-behind-hands youve disappoint paranormal activity insidious likely deliver area failed tell supposedly true story paranormal investigators aim rid family property suspect supernatural visitations disprove turn creaky floorboard slam doors tackle head-on leap faith require buy theme youre okay play pretty genre confines particular incident bring deal describe surround spirit malevolent hide public fairly amityville-like circumstances family new house discover basement sealed boarded-up doorway surprise follows decide look basement surprise james wan manage tire theme haunt possession revive convincingly strange kind truly tension terror enjoy sinister recently equally scary lose bite end conjuring tempo fairly satisfy conclusion turn comical middle minutes fire cylinder head finale intentional lure false sense security work beautifully youre type poo-poo genre general canst suddenly convert believer enjoy classic exorcista amityville poltergeist guarantee conjuring wont disappoint
22 0 4.4468 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre ism avid fan lately ive think isnt scare sinister skin appreciate james wants love saw insidious damn modern ghost story review state kinda lose momentum final act conjuring better scarier tense insidious ism gonna limb years classic rule making bad guy fully audience plus suspense tell throw water bottle screen sheer terror scare finally happened fake jump scares earn r rate blood sex profanity terror build scary masterpiece script acting direction style tone notch wants camera-work far movies choice set stroke genius feel authentic real story set present wouldnt effective scary style making costuming hair style throwback exorcist amityville extremely minor flaws length repetitiveness resolution happen fast dont care general flaws perfect havenet james wants conjuring ive time time scare badly dont miss amaze theater view experienced
23 0 5.6439 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre key conjuring freshness evidence ream fan argue internet site new table fan hungry time conjuring right splinter deal admire day age stand tall proud ream remakes sequel teen friendly slasher haunt multiplex frequency days free gore sex automatically alienate portion lustful member fan based serve appetising scare cauldron suspense deliver plenty particular table forget based true story tag kind irrelevant new technological age sell gimmick mean story true play bit regardless hoax charge embellishments buy premises commit scary story director james wan reward compliant story essentially base investigation early seventy paranormal specialist ed lorraine warren aid perron family victim dreadful supernatural event rhode island home wan build deftly let perron family live believe dream home start happen wan build slow instances create palpable sense dread camera work intelligent moment maximum impact yearn warrens involved kill boo-jump scares place care marry superbly mount tension naturally clich convention haunt house strange smells creaky doors ominous cellar supplement wants talented knack scare effective production designs mysterious bruise literal leg pull breath hold game hide seek bona fide pant soil moments conjuring lesson sustain unease finale unleashed script devoid cheese pointless filler refresh sub-genre suffer problems joseph bishara musical score absolute nerve shredder refresh accompaniment doesnt resort telegraph shriek tell afraid overwhelm scene john leonetti cinematography gothic textures house outside lakeside farmhouses strong lead cast vera farmiga patrick wilson lili taylor ron livingston trump met critical box office success conjuring justify reputation superb haunt house
24 0 5.0902 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre conjuring high class hard scare care character story compel feel interest time based true life events ed lorraine warren paranormal investigator set help family terrorize demon terrify case live hadnt share love place 70 agree completely director james wants point view impossible set present example teenage daughter haunt family picture demon diphones post post instagram basically demon turn victim human bad guys isnt perfect thought plot hole scare jump seat build tension let character start craziness set 70 feel 70 slow zoom style wouldnt expect modern believe james wan outdo ill far scary ive life thank james wan luck fast furious d recommend look guess feel audience feel watch exorcist dont look poltergeist time
25 0 3.2186 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre pit human horrendous extra terrestrial end cheap imitation aliens series pitch black stand fine piece sci-fi excellent favorite aspect lighting beautifully employ different colors shade intensity light set mood lend unique feel different normal light generally subject vin diesel bring character life excellent manner skillfully avoid routine portrayal harden criminal riddick diesel character personal journey thankfully vin doesnt drop ball remainder cast exception talented gorgeous claudia black unknown turn marvelous performances animate diverse character unique quirk mannerisms pitch black perfect example resource excessive budget special effect adequate time mean sole focus high budget blockbusters use science fiction medium tell engaging provoke story tell mediocre story use flash science fiction
26 0 4.0486 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre pitch black survival story survive hostile alien world hostile enemies task difficult near enemy survive group plot pitch black usual time different variations shine well-written characters group consist different people interest jacks boy secret fryd pilot hard time conscience johns bounty- hunter drug-habit imams holy man face fact god cruel riddick convict murderer learn value characters start live arendt paper genre start care characters reddick feel odd suppose bad guy line light pitch black think character familiar work casting rarely actor fit role radha mitchell suitable fryd cole hauser bring right cruelness sense responsibility johns impressive work vin diesel job riddick hand reddick creepy definitely dangerous deep character suppose survive pitch black friend close enemy closer
27 0 1.8767 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre admit fan serenity reddick chronicles bite predetermine anticipation flip ism vin geisel fan somewhat mono-dimensional range characters embodiment riddick fit perfectly story play pace encumber gratuitous special effect fit cgi appropriate glare technical defect story start walk breathe air slight concern difference atmosphere microbe common flaw gloss stories canst harp detail radha mitchell smoking hot ridiculous frivolous tough season interplanetary merchant marine ships captain character fry nail perfectly remind andrea start hungarian assassin gilda tigers lady appear drop check heartbeat digress cases trouble grant star pitch black
28 0 4.7084 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre doubt excite satisfy ive years plot print banal- ship crash desert planet suns survivor adjust landscape darkness fall monster appear pilot fryd moment cowardice descent atmosphere jettison passengers charge group enlist help convict murderer reddick lead darkness escape ship surgically enhance eye dark simple character complex three-dimensional conflict trait whoas whoas bathe performance uniformly superb radha mitchell fry steel leadership overcome fears prevent bloodshed cole hauser man reddick custody agendum idiosyncratic standards belong vin diesel reddick magnetic screen presence ive years face shadow doesnt deal draw attention draw attention speak moving diesel doesnt trivialise character easily heart gold reddick mean vicious man approach ship let glimpse tentative step care self care technically light effect excellent end spectrum overbright triple sunlight pitch darkness special effect riddick monsters point view add suspense sound effect monster fly ultrasound monster anatomically plausible suitably frightening editing tight jar time literally pad fades time re-orient open shoot end credit wit scene line dialogue single camera shoot important time understand ally caveat science solar system model impossible planets revolve suns vice versa doesnt affect human story havenet point
29 0 1.5165 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre open scene pretty incredible ive numb sci-fi special effect roommate look open sequence plainly sensory overloaded plot pretty simple nice sci-fi let audience figure technology instead waste time subtly explain creature interesting dont look overall entertain
In [88]:
lsi_most_representative_docs = get_most_representative_docs(lsi_dom_topics_final, n_topics=20, lsi_flag=True)
lsi_most_representative_docs.style.applymap(color_green).applymap(make_bold)
Out[88]:
Probability_Contribution Topic_Keywords Most_Representative_Document
Topic_Number
0 12.0119 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre id eye years constantly check released trailer immediately hook need finally safely wait worth review point critic keyboard warrior complain baskin low plot hearing complaint grind gears devoid creativity generate interest plot basking designs dont fine low plot complaint absurd think masterful classic threadbare plot bat eyelash fact praise love basking writer director eternal craft wonderfully nightmarish understand fabric bad dream- dream wake canst baskin sense scene scene follow narrative plot thread detail stick thoroughly dreamlike quality dive deep dream realms character deliver spooky speech fate death camera fetishizes detail look irrelevant eye draw scour scene clue hint lurk shadows scene atmospheric moody uneasy vibe happening simple plot doesnt mean simple leave open imagination think concept marry usually mean killer monster isnt seen gory stuff happen screen true baskin let use imagination story theres obviously complex history antagonist think unnerve scary watch kill victims speak victims time protagonists team police officers happen stumble lair old abandon police station- clearly theyve small cult kill dozen dozen people extremely brutal ritualistic manners ive prop filmmakers create disturb look cult ive seen perfect cast cult leader meet cerrahoglu obviously wont mean audience mehmet screen credit god performance deeply unsettle extremely disturbing bright dark future movies new berryman love fondness genre actor humanly possible steal establish lead point meet dynamic chill screen presence evernol inspiration readily apparent genre fans clive barkery eli roth david lynch nicolas winding refn- manage blend eerie euro art-house vibe hardcore splatter flick vibe no- scratch doesnt blend them- riskier eerie euro art-house cinema smash rub face anxiety inducing gross- extreme gorer average splatter flick baskin bloody insane i- genre fan jaded hell doesnt mean canst recognize dement wild ride sick haunt concern mood instantly disturb imagery backstory plottings necessary story bit tell character interactions fine aspect baskin succeed overall excel leave memorable image average person wouldnt want let stick head baskin doesnt look cheap low budget immersive gritty readily apparent flaw chalk differ taste opinions people plot-heavy movies obviously let baskin bit doesnt mean thin plot flaw retrospect feel small locations restaurant highways old not-so-abandoned police station feel small happen location milk set surprisingly effective lesser hands ve absolute misfire slick piece gore-splatter cinema moody cinematography vibrant colors synth heavy score bring mind extra bloody satanic spin-dry wrong turn nicolas winding refund couldnt bad tried baskin destine overnight cult status genre immortality love
1 -47.9122 sarah, juno, friend, cave, subconscious, action, deep, leave, stab, conscious, level, people, kill, event, dreamer, wreck, husband, story, run, death whats interest deep mean symbol women sarah friend juno open sarah lose husband child horrific wreck course apparent accident cause husband distract upset affair juno sarah aware subconsciously juno betrayal lead death husband child place mentally emotionally acknowledge conscious level sarah awaken wreck sequence run darkness darkness symbolize action sarah reject run isnt ready classically chase dream dreamer pursue subconscious horrible form dreamer conscious effort stop run speak scary chase dreamers unconscious speak directly dreamer conscious dreamer drop ego defense running interact subconscious chasers gain new insight year pass juno friend spelunking lie cave discover want sarah cave complex juno sick mind sheds perform type penance doesnt sheds manipulate friend incredibly dangerous juno manipulator narcissistic point view people real juno feel going underground dream imagery subconscious basement bar fight club underwater sarah return darkness run earlier start experience hallucination waking dreams mainly daughters laugh subconscious pull event damage psyche wreck trap deep cave sarah deep deep subconscious readily apparent deep point cave deep level consciousness encounter cave people watch ll notice stop travel downward fall push downward choose cave people base ancient level sarah mind lack senses problem-solving tool-using skill strong aggressive primal force creatures key event happen juno sarah friend begin pick juno true character revealed stab friend accident leave help shock purpose whats interest soon juno reunite main group wont leave sarah missing act differently people character arendt watching juno person weak character key event sarah separate group end den cave dwellers heart low level subconscious juno original betrayal fully sarah conscious self friend juno stab bleed death tell sarah juno husband low level sarah understand juno juno type woman wreck marriage friend stab friend leave die lead friend certain death ego trip short juno destructive force sarah kill hurt friend end suffering action juno refused sarah empathize friend refuse leave pain escape juno juno feel feeling safety important friends suffering event happen room sarah kill cave dweller child observe mother grieve loss sarah commit sin juno mother kill sarah miss sarah life sarah need vengeance juno sarah eventually meet juno begin survivors fight cave dwellers sarah confront juno lies manipulations betrayals sarah stab juno leg leave cave dwellers action juno early stab friend leave different sarah conscious choice kill juno responsibility action juno avoid responsibility sequence sarah climb upward dark light symbolic return consciousness real world parallel hamlets miss sarah miss hamlets need action horribleness action recoil spend story point term
2 35.0989 people, battle, story, world, royale, school, youth, high, island, kill, idea, juno, sarah, videogame, hate, love, young, teenagers, die, problem kenji fukasaku battle royale 2000 plenty past ive apart battle royale ism search battle royale affect people rabid fan battle royale people hate let tell battle royale exercise right explore idea idea poorly realized battle royale awesome hypothetically traumatic happen teenagers dont focus happen group high school student send abandon island kill bring bizarre idea fruition include civil unrest teenage anxiety nation literally terrorize youth set japan hit pretty close reality current problem japanese youth fact poorly receive government fear release incite riot act mayhem youth focus problem world young people volatile 20-30 year ago battle royale capture essence todays youth face circumstances friends kill friend bully survive time live videogame love play home note counter-strike half-life popular videogame mod example easily prepare young people reality weapons bullet clip mp5 assault rifle sound questions easily answer videogame today weapon street world hand child young years-old videogame set create comfortable experience weapons videogame necessarily people want gun familiarity guns mention love play counter-strike continue play future dont hate game ism point present fairly realistic portrayal weapons problem survivor island massacre add extra pressure unprepared child fight lives truly shock actor actress select portray teen age characters arendt age 20-30 somethings happen look young literally teenagers flick bite comment live century time frequently fear country outside forces terrorism forefront average north americanos mind world trade center attack cents endless coverage event easily problem international events headline grabber focus young people fill `rage unleash anger helpless peer array weapon mainly guns school shooting shock world child start kill peers battle royale mean trivialize school shooting youth violence examination length government order discipline youth ludicrous idea character stay true form profess hold crush dye breath naively trust theyve admire popular kids sick strange beautiful familiar different completely engaging people feel plot silly dialogue hammy nonsense time naysayer praise braveheart honest portrayal scotland historical hero love braveheart think bogus certain battle event happen william wallace man look rape kill woman small child kind feel sell wallace hero battle royalet draw fictitious character plot far interest think life kill friend high school stick island death day refuse answer question sicken think feel disturb kid battle royalet whats wrong pick lottery end island japan exist battle royalet year random high school class pick events lead believe youth japan bad seed doesnt apply class follows intent purposes innocent dialogue character poignant real totally innocent literally limit vocabulary understand world furthermore mention earlier character profess love classmate love high school weird time anybody awkward time experience misunderstandings people high school learn truth people think high school theres social wall stop real open communication people island force unchecked emotion feeling flow character death horizon label dialogue lousy circumstances obviously intelligent well-organized people world exist high school teenager brash foolish irresponsibly reckless theyve learn experienced rarely experienced teenagers island kill learn new wont matt consider theyll soon dead naturally insane mutter math equation teacher promise valuable real world feel need fulfill sexual desires want die virgin right situation spend hour alive civilize possible purpose game affect teenagers hurry battle isnt finish day die easy people charge low-jacked teenager collar explode head clean default explosion open jugular bleed die hope dream future grisly doesnt specifically cater gore hounds disturb death scene moment idea shock theres need excessive fact upset want bloody parade carnage reason form entertainment exist idea sicken compel satisfy layer visual end street sceptic people unable maturely grasp concept people hate canst blame hollywood dominant authority filmmaking world greatly expressive thought-provoking medium trite bore things recycle repackaged re-sold re-distributed point people hardly accept new radical different safe generics commercial reason existence appear highly questionable battle royale isnt change world wish damage place challenge socio-political norm subtitles thats alright matt dont press attention leonardo dicaprio enjoy battle royale kenji fukasaku direct adapt screen son kenta able rest peace man die january 12th 2003 years-old want died wish fan don t hate matt hard theres better j symister 2002
3 -23.0985 water, ring, dark, original, yoshimi, watch, scene, work, feel, world, girl, ikuko, think, time, beast, child, nakada, young, form, people yoshimi matsubara hitomi kuroki middle nasty divorce husband unio hamada fumiyo kohinata issue contention daughter ikuko ario kanno unio accuse yoshimi unstable point yoshimi award temporary custody ikuko apartment ikuko live pick less-than-ideal apartments affordable soon strange occurrence begin yoshimi bedroom ceil develope water stain mysterious puddle water appear different locations unusual item appearing despite attempt discard yoshimi periodically strange girl glimpses ikuko begin act oddly yoshimi work sheds trouble balance care ikuko things spiral control problem yoshimi divorce sinister supernatural despite dark waters relatively overt similarity numb filmic works director hide nakata successful films--at fame ring 1998 better awfully close dark water easily raise score subsequent viewings facet open example fact check shortly write review re-watched beginning open credit extremely eerie impact doesnt hit youve fully realize youre look watch shot similarity include thematic resemblance ring shouldnt surprise consider nakada director base novel man japanese stephen king japanese press--koji suzuki ring dark waters menace form young haired japanese girl frequent mysterious appearances girls focus irony--they suppose cute turn performance adult fellow cast innocent girl menace unnerving menace accompany water water important symbolism ring took venture guess nakada andor suzuki fear water impersonal took water powerful force easily adapt surrounding easily mold permeate world visual symbol kamis shinto essence beingness permeate things godlike soul dead human tsumi pollution form kami symbolically cleanse water important symbolic commonality share ring dark water claustrophobic spaces occur ring form closet crawl spaces dark water elevator structure ll realize importance near end water combine elevator enable nakada nice nod stanley kubrick shining 1980 scene similarity ring dark water concern familial problem concern fact symbolic metaphor familial problem ringu ring clear nakata latest american ring film--the ring 2005 feature young mother struggle maintain normal existence child dark water easy element mere metaphor yoshimi psychological decline effect daughter echo problematic childhood--we learn parent divorce young open dramatic scene yoshimi child wait school pick hear comment mother bad dark water focus nakata deliberate pace perfect spooky event subtle unnerving nakada achieve amaze build-ups scene elevator near end frighten reveal reveal work nakada build tension stretch pregnant pause viewer ready burst scene dark water succeed story relatively simple straightforward unlike typical american story told implication viewer frequently leave figure decision event base seemingly innocuous comment antecedent scene follow relationship scenario change follow scene words assumption happened sound complex aim wonderfully achieved simplify event screen famous asian dream logic present supernatural events doesnt usurp plot continue gradually hone build tension yoshimi husband ikuko mystery girl apartment complex ending comment element profound theyve changed uncanny poignant
4 -17.745 story, original, room, work, beast, vampire, ghost, 1408, battle, watch, french, school, character, scene, remake, le, creature, frontal, mike, genre 1765 stalk mountain south-western france beast pounce human animal terrible ferocity beast notorious king france dispatch envoy happen kill creature end beast gevaudan kill 100 people day entirely wolf hyena supernatural shepherd life-expectancy red-suited guy star trek beast popular myth france albeit root firmly reality somewhat surprisingly outside world incredibly categorising le pace des loups tricky ill period costume martial-arts werewolf surprisingly piece work provide dont concentrate hard previously period costume martial-arts werewolf mystery expect plenty imitation future taking beast start point quickly diverge historical fact step pace introduce heroes gregoire frontal samuel le bihan mani mark dacascos midst torrential storm culminate magnificently stage fights frontal dispatch king beast frontal represent new rational world enlightenment force confront backward superstitious france outside capital mania iroquois shaman hunter befriend frontal whilst adventure americas bring type wisdom entirely time america dark mysterious place home fear europeans course shortly home republicanism sweep france remake old world new image le pace des loups wear republican colour sleeve use conflict rationalism stereotypical backward villager drive home point old-fashioned territory source plot guvaudan sort village inhabitant sleepy hollow creeps england christopher lee lord manor peter cushing priest le pace des loups strong french language cast possible mix veteran talent dominate priest sardis jean-fran stévenin saturnine jean francois vincent cassel cripple hunter explorer rapidly dangerous beast scornful change paris seek shield world future remainder population stupid indolent superstitious evil hold new rational world cities beast extension physical monster beast projection villager hatred bigotry strong female role unusual movies different characters strikingly elegant hypnotic courtesan sylvia monica bellucci play role seductress frigid professionalism world woman wit protect dangerous far appears unsure help hinder heroes mysterious captivating complete contrast innocent fragile astonishingly beautiful madeline emilie dequenne young sister protective jean francoise surrounded evil prejudice superstition clearly romantic heroine intend represent french republic symbol frontal fall hopelessly love witty charm woman risk conflict jean francois lead fantastic share chemistry reminiscent relationship butch sundance le frontal wise needed sensational think mani human mani man word utterly dominate screen present needless fantastic fighters live die creature fortunately delivers wisely chance look animal nasty viewers mind fill details creature surprisingly sparingly viewer expect pounce doesn t minute appear wonderful shock stuff reminiscent alien whilst design animal creature workshop perfect cgi work standard expect couple daylight shot well-below nighttime work outstanding shoot creature stalk fog hero effective cgi work cinematically work bear resemblances ridley scotts gladiator luscious slow character-forming scene mix frantic camera work action scenes strange hybrid style work exceptionally frantic camera digital trick excellent effect flashback scene striking strongly solarised effect otherworldly texture designer previously involve merchant ivory production luxurious interior scene bite detail period piece special word costume use sumptuous fabric possible deal light candle fire light fill screen warm orange flesh tone miss chance flesh contrast exterior shot frequently chill blue wash hues french countryside look hostile world conceal form dark secrets countryside magnificently different stereotypical french landscapes tragically splendour play minuscule audiences people whilst queue american pie stretch auditoriums favour foreign language people think french cinema involve middle age peasant smoke gauloises whilst argue fine point philosophy revelation 140 minute run plot twist near end difficult accept wince accept doesnt outstay welcome english-speaking market subtitled sadly workmanlike translation witty dialogue isnt translated shame script poor french speaker sparkles numb misspelling grammatical error subtitle catch earlier read subtitles isnt art doesnt redefine genre doesnt preach nature irrational learn apart dont split group basement check light deliver hour solid entertainment ll silly grin face want finally word praise imaginative dissolve shot womans breast fade mountains doubt woman world eager christophe gans eiffel towering short le pace des loups paw
In [89]:
lsi_topic_dist = get_topic_distribution(lsi_dom_topics_final, lsi_most_representative_docs, n_topics=20)
lsi_topic_dist.style.applymap(color_green).applymap(make_bold)
Out[89]:
Documents_per_Topic Percent_of_Total_Corpus Topic_Keywords
Topic_Number
0 988 0.988 story, time, character, people, think, feel, watch, scene, end, dont, work, plot, look, want, doesnt, play, love, director, start, genre
1 4 0.004 sarah, juno, friend, cave, subconscious, action, deep, leave, stab, conscious, level, people, kill, event, dreamer, wreck, husband, story, run, death
2 4 0.004 people, battle, story, world, royale, school, youth, high, island, kill, idea, juno, sarah, videogame, hate, love, young, teenagers, die, problem
3 2 0.002 water, ring, dark, original, yoshimi, watch, scene, work, feel, world, girl, ikuko, think, time, beast, child, nakada, young, form, people
4 2 0.002 story, original, room, work, beast, vampire, ghost, 1408, battle, watch, french, school, character, scene, remake, le, creature, frontal, mike, genre
In [90]:
#token pattern prevents default splitting of hyphenated words
vectorizer = CountVectorizer(token_pattern='(?u)\\b[\\w-]+\\b', analyzer='word', max_features=5000)
x_counts = vectorizer.fit_transform(all_reviews_joined)

transformer = TfidfTransformer(smooth_idf=False)
x_tfidf = transformer.fit_transform(x_counts)
In [91]:
lsi_svd_2D = TruncatedSVD(n_components=2, algorithm='randomized', n_iter=500, random_state=222)
lsi_2D = lsi_svd_2D.fit_transform(x_tfidf.T)
In [92]:
svd_2D_scatter(lsi_2D, color_scale_dimension=1)
In [93]:
lsi_svd_3D = TruncatedSVD(n_components=3, algorithm='randomized', n_iter=500, random_state=222)
lsi_3D = lsi_svd_3D.fit_transform(x_tfidf.T)
In [94]:
svd_3D_scatter(lsi_3D, color_scale_dimension=0)
In [95]:
#token pattern prevents default splitting of hyphenated words
vectorizer = CountVectorizer(token_pattern='(?u)\\b[\\w-]+\\b', analyzer='word', max_features=5000)
x_counts = vectorizer.fit_transform(all_reviews_joined)

transformer = TfidfTransformer(smooth_idf=False)
x_tfidf = transformer.fit_transform(x_counts)

xtfidf_norm = normalize(x_tfidf, norm='l1', axis=1)
In [96]:
mod_list_nmf_umass, coh_list_nmf_umass = compute_coherence_values(NMF, dictionary, all_reviews_text, max_topics=20, 
                                                                  transformed_vectorizer=vectorizer, tfidf_norm=xtfidf_norm,
                                                                  min_topics=2, stride=3, n_top_words=20, measure='u_mass', 
                                                                  nmf_flag=True)

mod_list_nmf_cv, coh_list_nmf_cv = compute_coherence_values(NMF, dictionary, all_reviews_text, max_topics=20, 
                                                                  transformed_vectorizer=vectorizer, tfidf_norm=xtfidf_norm,
                                                                  min_topics=2, stride=3, n_top_words=20, measure='c_v', 
                                                                  nmf_flag=True)
Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'sklearn.decomposition.nmf.NMF'> using the U_MASS measure


Coherence of model with 2 topics has been computed
Coherence of model with 5 topics has been computed
Coherence of model with 8 topics has been computed
Coherence of model with 11 topics has been computed
Coherence of model with 14 topics has been computed
Coherence of model with 17 topics has been computed
All coherence values have been computed for the <class 'sklearn.decomposition.nmf.NMF'> using the C_V measure


In [97]:
best_num_nmf_topics = show_best_num_topics('NMF', coh_list_nmf_umass, coh_list_nmf_cv, max_topics=20, min_topics=2, stride=3)
print('The most coherent number of NMF topics to use is: {}'.format(best_num_nmf_topics))
The most coherent number of NMF topics using the U_MASS measure is: 17
The most coherent number of NMF topics using the C_V measure is: 2


The most coherent number of NMF topics to use is: 2
In [98]:
nmf = NMF(n_components=best_num_nmf_topics, init='nndsvd', random_state=222)

nmf.fit(xtfidf_norm)
Out[98]:
NMF(alpha=0.0, beta_loss='frobenius', init='nndsvd', l1_ratio=0.0,
  max_iter=200, n_components=2, random_state=222, shuffle=False,
  solver='cd', tol=0.0001, verbose=0)
In [99]:
nmf_df = get_nmf_topics(nmf, best_num_nmf_topics, 20)
nmf_df
Out[99]:
Topic # 01 Topic # 02
0 watch summary
1 story describe
2 dont funny
3 time scary
4 feel plot
5 people story
6 think phobia
7 end salvage
8 character backpackers
9 original ward
10 scare gory
11 ive bias
12 ism end
13 look insidious
14 scene gods
15 fan gift
16 love plenty
17 leave peter
18 scary novice
19 act purikitpanya
In [100]:
nmf_hellinger_topic_distances = get_most_similar_topics(nmf, nmf_flag=True, columns=nmf_df.columns)
pretty_print(nmf_hellinger_topic_distances)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.8287134542069157, 'Topic # 01', 'Topic # 02')

______________________________________________________________________

NMF: Second Pass with U_Mass Coherence optimized number of components

In [101]:
nmf_2 = NMF(n_components=17, init='nndsvd', random_state=222)

nmf_2.fit(xtfidf_norm)
Out[101]:
NMF(alpha=0.0, beta_loss='frobenius', init='nndsvd', l1_ratio=0.0,
  max_iter=200, n_components=17, random_state=222, shuffle=False,
  solver='cd', tol=0.0001, verbose=0)
In [102]:
nmf_df_2 = get_nmf_topics(nmf_2, 17, 20)
nmf_df_2
Out[102]:
Topic # 01 Topic # 02 Topic # 03 Topic # 04 Topic # 05 Topic # 06 Topic # 07 Topic # 08 Topic # 09 Topic # 10 Topic # 11 Topic # 12 Topic # 13 Topic # 14 Topic # 15 Topic # 16 Topic # 17
0 feel summary gods zombies vs insidious ghost phobia ism pirate james ring hill evil conjuring wrong fun
1 character describe gift morality man whats sci-fi segment think poses watch ju-on silent remake wow downhill jar
2 story funny peter mandatory empathy scary background funny ive waste split scary game pure scare answered entry
3 time scary plenty long-haired foreign aftermath people kinda review significance dont owe look need jump don hollywood
4 scene plot debut played humanity magically complicate backpackers eden disease personalities exorcista think yearn cinema boring sundance
5 end story zombie beautiful train grab depend salvage thrillers trust mcavoy agree mystery vampire sequel rating park
6 real recommend end train eyes intense creativity ward watch wood different reviews isnt poetic conjure t seriously
7 people life story woman remember scared special suspense worth bunch saw powerful drama thought-provoking ive m style
8 genre korean infect means treat risk magnificent gory read truth award water doom profound incredible list redundant
9 director interesting suspenseful girl eye door relate bias ill answer wait uncomfortable visuals gems boredom hole inferior
10 new love virus cheap mind recently asian stories actress question win unlike resident achieve youve predictable day
11 twist strongly corpses short excellent jump effect story better wouldnt comment supernatural said contrast anymore couldnt abrams
12 act disturb zombies idea definitely entirely expectation close bad kill played stuff crowd watch lucky night innovative
13 didnt dont chainsaw original original hear comment messy fan slow mistake cheap video artistic teenager large importantly
14 doesnt works texas play enjoy genuine hollywood novice slow night today shot welcome explore lets expect unnecessary
15 house troll slight address young talk dont end thriller happen wont decide crap dead disappoint question remakes
16 family polish massacre army act fail enjoy humorous ending scary performance experience visual hindi honest minute paul
17 work ideal flesh hat look scenes typical cool build dont shyamalan dark pop nature usually leave ugly
18 want attractive quietly overwhelm time remind terrify scary die people convince watch intelligent novel scary isnt pretty
19 tension watched random leg story room believe marsha hard time write ism memorable somewhat couldnt wait red
In [103]:
nmf_hellinger_topic_distances_2 = get_most_similar_topics(nmf_2, nmf_flag=True, columns=nmf_df_2.columns)
pretty_print(nmf_hellinger_topic_distances_2)
Hellinger Distances between topics are: 

______________________________________________________________________


(0.9584644507879246, 'Topic # 03', 'Topic # 06')
 (0.9424645212814688, 'Topic # 03', 'Topic # 15')
 (0.9399235670404483, 'Topic # 02', 'Topic # 05')
 (0.9346295431141141, 'Topic # 03', 'Topic # 05')
 (0.9344291361702227, 'Topic # 03', 'Topic # 13')
 (0.9322648514116035, 'Topic # 04', 'Topic # 06')
 (0.9315124306651026, 'Topic # 02', 'Topic # 04')
 (0.9301052202804005, 'Topic # 03', 'Topic # 16')
 (0.9294710886755139, 'Topic # 08', 'Topic # 10')
 (0.9291101049170272, 'Topic # 03', 'Topic # 07')
 (0.9291003746279596, 'Topic # 05', 'Topic # 10')
 (0.9289175001415316, 'Topic # 04', 'Topic # 16')
 (0.9282824812843677, 'Topic # 04', 'Topic # 13')
 (0.9281640990078213, 'Topic # 04', 'Topic # 10')
 (0.9258252741428218, 'Topic # 06', 'Topic # 17')
 (0.9251840639484765, 'Topic # 03', 'Topic # 14')
 (0.924785695848468, 'Topic # 06', 'Topic # 08')
 (0.9209535866682126, 'Topic # 03', 'Topic # 08')
 (0.9199740730254865, 'Topic # 08', 'Topic # 16')
 (0.9196165653244697, 'Topic # 03', 'Topic # 10')
 (0.9193324527539374, 'Topic # 02', 'Topic # 03')
 (0.9192197190729483, 'Topic # 02', 'Topic # 10')
 (0.917734398614864, 'Topic # 02', 'Topic # 14')
 (0.9173162625447314, 'Topic # 03', 'Topic # 17')
 (0.9168218643907617, 'Topic # 02', 'Topic # 16')
 (0.9161233622399915, 'Topic # 04', 'Topic # 15')
 (0.9133152848397521, 'Topic # 03', 'Topic # 09')
 (0.9116331138366993, 'Topic # 02', 'Topic # 06')
 (0.911593557277736, 'Topic # 08', 'Topic # 13')
 (0.9114488871983222, 'Topic # 03', 'Topic # 11')
 (0.9113735217700517, 'Topic # 08', 'Topic # 12')
 (0.908458108265172, 'Topic # 02', 'Topic # 15')
 (0.9064553140855598, 'Topic # 03', 'Topic # 12')
 (0.9062050177358214, 'Topic # 04', 'Topic # 08')
 (0.9059031502277907, 'Topic # 10', 'Topic # 17')
 (0.9058733302866112, 'Topic # 04', 'Topic # 12')
 (0.9056778335861626, 'Topic # 05', 'Topic # 06')
 (0.9055958689354423, 'Topic # 08', 'Topic # 14')
 (0.9044965853974102, 'Topic # 02', 'Topic # 07')
 (0.9043340941387561, 'Topic # 04', 'Topic # 09')
 (0.903994283672788, 'Topic # 03', 'Topic # 04')
 (0.9038715444306065, 'Topic # 05', 'Topic # 16')
 (0.9037995829889024, 'Topic # 13', 'Topic # 17')
 (0.903413811906169, 'Topic # 04', 'Topic # 05')
 (0.9020537456963644, 'Topic # 02', 'Topic # 11')
 (0.901835879590631, 'Topic # 04', 'Topic # 07')
 (0.9012207668445605, 'Topic # 05', 'Topic # 08')
 (0.8999494677207568, 'Topic # 04', 'Topic # 11')
 (0.8990402272302405, 'Topic # 10', 'Topic # 11')
 (0.8980293628586249, 'Topic # 15', 'Topic # 17')
 (0.897705789738865, 'Topic # 02', 'Topic # 12')
 (0.897303524108698, 'Topic # 05', 'Topic # 11')
 (0.8971279939937775, 'Topic # 06', 'Topic # 11')
 (0.8969129114284167, 'Topic # 04', 'Topic # 17')
 (0.8967706373411575, 'Topic # 08', 'Topic # 11')
 (0.896114020736355, 'Topic # 05', 'Topic # 12')
 (0.8944365471996264, 'Topic # 02', 'Topic # 13')
 (0.8939917871520082, 'Topic # 07', 'Topic # 08')
 (0.892794574546296, 'Topic # 02', 'Topic # 17')
 (0.89276442270781, 'Topic # 02', 'Topic # 09')
 (0.8925193082712505, 'Topic # 12', 'Topic # 17')
 (0.8921381443325341, 'Topic # 08', 'Topic # 17')
 (0.8901026918028777, 'Topic # 06', 'Topic # 14')
 (0.8889367919218137, 'Topic # 05', 'Topic # 13')
 (0.88844179547972, 'Topic # 16', 'Topic # 17')
 (0.888017458625833, 'Topic # 06', 'Topic # 13')
 (0.8877450620813365, 'Topic # 10', 'Topic # 14')
 (0.8865843692498548, 'Topic # 07', 'Topic # 16')
 (0.8865535168589743, 'Topic # 05', 'Topic # 14')
 (0.8859040418084831, 'Topic # 10', 'Topic # 15')
 (0.8854040099500482, 'Topic # 13', 'Topic # 16')
 (0.8849448321471554, 'Topic # 06', 'Topic # 16')
 (0.8846477792497829, 'Topic # 04', 'Topic # 14')
 (0.8839170558595878, 'Topic # 08', 'Topic # 15')
 (0.8826106830401623, 'Topic # 13', 'Topic # 14')
 (0.8825175820868869, 'Topic # 05', 'Topic # 17')
 (0.8822830273699803, 'Topic # 05', 'Topic # 15')
 (0.8817729623835481, 'Topic # 08', 'Topic # 09')
 (0.8815908166368477, 'Topic # 07', 'Topic # 11')
 (0.8814407765137757, 'Topic # 07', 'Topic # 14')
 (0.8810655361398263, 'Topic # 11', 'Topic # 16')
 (0.881007864900373, 'Topic # 14', 'Topic # 16')
 (0.8805133571218197, 'Topic # 11', 'Topic # 13')
 (0.8794852605253304, 'Topic # 06', 'Topic # 09')
 (0.8769534447500179, 'Topic # 11', 'Topic # 17')
 (0.8769305197381267, 'Topic # 05', 'Topic # 07')
 (0.8764848171790806, 'Topic # 07', 'Topic # 17')
 (0.8761736952013941, 'Topic # 07', 'Topic # 10')
 (0.8741326480827946, 'Topic # 06', 'Topic # 07')
 (0.8740462766943192, 'Topic # 12', 'Topic # 14')
 (0.8737575271325964, 'Topic # 10', 'Topic # 12')
 (0.8737189878294761, 'Topic # 14', 'Topic # 15')
 (0.873350406073119, 'Topic # 15', 'Topic # 16')
 (0.8719561260038503, 'Topic # 07', 'Topic # 13')
 (0.8698688361693404, 'Topic # 12', 'Topic # 13')
 (0.8689094667820856, 'Topic # 07', 'Topic # 12')
 (0.8685083498274481, 'Topic # 11', 'Topic # 12')
 (0.8682413384257885, 'Topic # 12', 'Topic # 16')
 (0.867391360926769, 'Topic # 10', 'Topic # 13')
 (0.8653900439144379, 'Topic # 09', 'Topic # 10')
 (0.8624647540171855, 'Topic # 06', 'Topic # 10')
 (0.8623928015184431, 'Topic # 11', 'Topic # 15')
 (0.8609140296129003, 'Topic # 14', 'Topic # 17')
 (0.8605266168357723, 'Topic # 09', 'Topic # 13')
 (0.8598861235951925, 'Topic # 09', 'Topic # 17')
 (0.8597004573353824, 'Topic # 07', 'Topic # 09')
 (0.8578453645250446, 'Topic # 09', 'Topic # 14')
 (0.857427955680357, 'Topic # 06', 'Topic # 12')
 (0.8571208200575151, 'Topic # 13', 'Topic # 15')
 (0.8561277260411315, 'Topic # 10', 'Topic # 16')
 (0.8554387815110195, 'Topic # 07', 'Topic # 15')
 (0.8550259091228346, 'Topic # 11', 'Topic # 14')
 (0.8531370139921345, 'Topic # 02', 'Topic # 08')
 (0.851784942370877, 'Topic # 01', 'Topic # 03')
 (0.8499604120234221, 'Topic # 06', 'Topic # 15')
 (0.8499454529807071, 'Topic # 01', 'Topic # 02')
 (0.8484159121000783, 'Topic # 09', 'Topic # 16')
 (0.8483299547608233, 'Topic # 09', 'Topic # 15')
 (0.8444322887790761, 'Topic # 05', 'Topic # 09')
 (0.8442968743292932, 'Topic # 09', 'Topic # 11')
 (0.8384120417252762, 'Topic # 09', 'Topic # 12')
 (0.8328400557963762, 'Topic # 12', 'Topic # 15')
 (0.8296730150866366, 'Topic # 01', 'Topic # 04')
 (0.8267393167070708, 'Topic # 01', 'Topic # 08')
 (0.8178112828683857, 'Topic # 01', 'Topic # 06')
 (0.8174981747238305, 'Topic # 01', 'Topic # 10')
 (0.7956580202009549, 'Topic # 01', 'Topic # 05')
 (0.7947136346996182, 'Topic # 01', 'Topic # 16')
 (0.7867317183203042, 'Topic # 01', 'Topic # 12')
 (0.7826714712806262, 'Topic # 01', 'Topic # 07')
 (0.7747618350774251, 'Topic # 01', 'Topic # 17')
 (0.7717195114488037, 'Topic # 01', 'Topic # 13')
 (0.7695156200854026, 'Topic # 01', 'Topic # 15')
 (0.7671204907764337, 'Topic # 01', 'Topic # 11')
 (0.7526477622365153, 'Topic # 01', 'Topic # 09')
 (0.7507159579845191, 'Topic # 01', 'Topic # 14')

______________________________________________________________________
In [104]:
svd = TruncatedSVD(n_components=2)
documents_2d = svd.fit_transform(x_tfidf)
 
df = pd.DataFrame(columns=['x', 'y', 'document'])
df['x'], df['y'], df['document'] = documents_2d[:,0], documents_2d[:,1], range(len(all_reviews_text))
 
source = ColumnDataSource(ColumnDataSource.from_df(df))

#alternative syntax
#source = ColumnDataSource({'x': np.array([documents_2d[:,0]]), 'y': np.array([documents_2d[:,1]]), 'document': np.array([range(len(all_reviews_text))])})

labels = LabelSet(x="x", y="y", text="document", y_offset=8,
                  text_font_size="8pt", text_color="#555555",
                  source=source, text_align='center')
 
plot = figure(plot_width=1200, plot_height=1000)
plot.circle("x", "y", size=12, source=source, line_color="black", fill_alpha=0.8)
plot.add_layout(labels)

#Cf. JSON Serialization issue: 
#    https://github.com/bokeh/bokeh/issues/5439
#    https://github.com/bokeh/bokeh/issues/6222
#   https://github.com/bokeh/bokeh/issues/7523
try:
    show(plot, notebook_handle=True)
except Exception as e:
    print('Note!: {}'.format(e.__doc__))
    print(e)
In [105]:
svd = TruncatedSVD(n_components=2)
words_2d = svd.fit_transform(x_tfidf.T)
 
df = pd.DataFrame(columns=['x', 'y', 'word'])
df['x'], df['y'], df['word'] = words_2d[:,0], words_2d[:,1], vectorizer.get_feature_names()
 
source = ColumnDataSource(ColumnDataSource.from_df(df))
labels = LabelSet(x="x", y="y", text="word", y_offset=8,
                  text_font_size="8pt", text_color="#555555",
                  source=source, text_align='center')
 
plot = figure(plot_width=1200, plot_height=1000)
plot.circle("x", "y", size=8, source=source, line_color="black", fill_alpha=0.8)
plot.add_layout(labels)

#Cf. JSON Serialization issue: 
#    https://github.com/bokeh/bokeh/issues/5439
#    https://github.com/bokeh/bokeh/issues/6222
#   https://github.com/bokeh/bokeh/issues/7523
try:
    show(plot, notebook_handle=True)
except Exception as e:
    print('Note!: {}'.format(e.__doc__))
    print(e)

LDA Visualization: T-distributed Stochastic Neighbor Embedding with PCA initialization

Cf. http://www.jmlr.org/papers/volume9/vandermaaten08a/vandermaaten08a.pdf

In [106]:
lda_base_model = lda.LDA(n_topics=10, n_iter=1000, random_state=111)
In [109]:
doc_x_tops = lda_base_model.fit_transform(x_counts)
In [110]:
#doc_x_tops.shape
In [111]:
#setting threshold above which documents allocated to topics are 'confidently' displayed

#threshold = 0.5

#idx of doc above threshold
#_idx = np.amax(doc_x_tops, axis=1) > threshold

#doc_x_tops = doc_x_tops[_idx]
In [112]:
# angle close to 1 prioritizes speed over accuracy
tsne_model = TSNE(n_components=2, verbose=1, random_state=222, angle=.1, init='pca')

# 10D -> 2D
tsne_lda = tsne_model.fit_transform(doc_x_tops)
[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 1000 samples in 0.001s...
[t-SNE] Computed neighbors for 1000 samples in 0.037s...
[t-SNE] Computed conditional probabilities for sample 1000 / 1000
[t-SNE] Mean sigma: 0.150693
[t-SNE] KL divergence after 250 iterations with early exaggeration: 69.865005
[t-SNE] Error after 1000 iterations: 0.866557
In [113]:
#20 colors
colormap = np.array([
    "#1f77b4", "#aec7e8", "#ff7f0e", "#ffbb78", "#2ca02c",
    "#98df8a", "#d62728", "#ff9896", "#9467bd", "#c5b0d5",
    "#8c564b", "#c49c94", "#e377c2", "#f7b6d2", "#7f7f7f",
    "#c7c7c7", "#bcbd22", "#dbdb8d", "#17becf", "#9edae5"
])

#number of displayed keywords
n_top_words = 10
In [114]:
#Cf. JSON Serialization issue: 
#    https://github.com/bokeh/bokeh/issues/5439
#    https://github.com/bokeh/bokeh/issues/6222
#   https://github.com/bokeh/bokeh/issues/7523
plot_tsne(all_reviews_joined, lda_base_model, vectorizer, doc_x_tops, tsne_lda, colormap, n_top_words=10)
In [ ]: